{"level":"info","ts":"2026-06-11T10:01:05.630Z","logger":"setup","msg":"Loaded feature gates","featureGates":{"RayClusterStatusConditions":true,"RayJobDeletionPolicy":false}}
{"level":"info","ts":"2026-06-11T10:01:05.630Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."}
{"level":"info","ts":"2026-06-11T10:01:05.630Z","logger":"setup","msg":"Setup manager"}
{"level":"info","ts":"2026-06-11T10:01:05.929Z","logger":"setup","msg":"NetworkPolicy controller registered (annotation-based activation)"}
{"level":"info","ts":"2026-06-11T10:01:05.929Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-11T10:01:05.929Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-11T10:01:05.929Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-11T10:01:05.929Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","msg":"starting server","name":"health probe","addr":"[::]:8082"}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":false}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-06-11T10:01:05.930Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-06-11T10:01:06.231Z","msg":"attempting to acquire leader lease opendatahub/ray-operator-leader..."}
{"level":"info","ts":"2026-06-11T10:01:06.239Z","msg":"successfully acquired lease opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.NetworkPolicy"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","msg":"Starting EventSource","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"}
{"level":"info","ts":"2026-06-11T10:01:06.240Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","msg":"Starting Controller","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","msg":"Starting Controller","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","logger":"controllers.RayCluster","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","msg":"Starting workers","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","logger":"controllers.RayService","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","logger":"controllers.RayJob","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","msg":"Starting workers","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","msg":"Starting Controller","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-11T10:01:06.829Z","msg":"Starting workers","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-11T10:03:50.214Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.214Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-vdznk"}
{"level":"info","ts":"2026-06-11T10:03:50.228Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.228Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e75aca7e-73de-4f0c-a25e-3bc5b6a962f2","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.228Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:03:50.229Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.229Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.229Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9"}
{"level":"info","ts":"2026-06-11T10:03:50.229Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9"}
{"level":"info","ts":"2026-06-11T10:03:50.236Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.236Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-vdznk"}
{"level":"info","ts":"2026-06-11T10:03:50.251Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","name":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:50.328Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e75aca7e-73de-4f0c-a25e-3bc5b6a962f2","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.328Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"508b204b-7e90-4936-b7ad-5d57d5f7ebe8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.329Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","name":"raycluster-auth-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:03:50.330Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.433Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.440Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"508b204b-7e90-4936-b7ad-5d57d5f7ebe8","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:03:50.527Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"508b204b-7e90-4936-b7ad-5d57d5f7ebe8","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:03:50.527Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"508b204b-7e90-4936-b7ad-5d57d5f7ebe8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.533Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.535Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","name":"kuberay-gateway-access","namespace":"test-ns-vdznk","operation":"created"}
{"level":"info","ts":"2026-06-11T10:03:50.547Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5"}
{"level":"info","ts":"2026-06-11T10:03:50.547Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:03:50.547Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.640Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.644Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","name":"test-ns-vdznk-raycluster-auth","operation":"created"}
{"level":"info","ts":"2026-06-11T10:03:50.727Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5"}
{"level":"info","ts":"2026-06-11T10:03:50.728Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:03:50.829Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5"}
{"level":"info","ts":"2026-06-11T10:03:50.829Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:03:50.829Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:50.829Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5"}
{"level":"error","ts":"2026-06-11T10:03:50.829Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"284f60a0-d7d1-4f3f-8c9a-617a7e1818d5","error":"Certificate.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:50.829Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"571b7c10-f16c-42f2-88d7-29157701e1de","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.927Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"571b7c10-f16c-42f2-88d7-29157701e1de"}
{"level":"error","ts":"2026-06-11T10:03:50.927Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"571b7c10-f16c-42f2-88d7-29157701e1de","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:03:50.927Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"571b7c10-f16c-42f2-88d7-29157701e1de","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:50.927Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"571b7c10-f16c-42f2-88d7-29157701e1de"}
{"level":"error","ts":"2026-06-11T10:03:50.927Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"571b7c10-f16c-42f2-88d7-29157701e1de","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:50.928Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0beb18bf-2001-451b-944b-3cd0ea36b369","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:50.928Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0beb18bf-2001-451b-944b-3cd0ea36b369"}
{"level":"info","ts":"2026-06-11T10:03:51.027Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a969022-47d8-4291-a2a6-9d0287d7e639","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:51.128Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a969022-47d8-4291-a2a6-9d0287d7e639"}
{"level":"info","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:03:51.252Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-vdznk","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9"}
{"level":"error","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e96d5c86-826f-4734-b50e-8a09418bf1a9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d"}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d"}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:03:51.337Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d"}
{"level":"error","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"58326c60-73d0-4184-b7b4-12913acf159d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631"}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631"}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:03:51.351Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-vdznk","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631"}
{"level":"error","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"2a8e3376-e6e8-4aa1-bd6d-7ea0f71e0631","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"186dec6e-2979-45ab-a9ed-7420b628d61f","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"8a32faed-d33a-466a-84e8-47ce60b2bc66","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"186dec6e-2979-45ab-a9ed-7420b628d61f"}
{"level":"error","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.428Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9"}
{"level":"error","ts":"2026-06-11T10:03:51.428Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"27e62ae9-c7fe-4eb4-8a4d-d735f87679b9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296"}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296"}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296"}
{"level":"error","ts":"2026-06-11T10:03:51.448Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b987eb1d-59f9-4ecc-b446-53d9608bb296","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a"}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a"}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a"}
{"level":"error","ts":"2026-06-11T10:03:51.529Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f25711d2-3881-4111-a46b-68c406cd569a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"8a32faed-d33a-466a-84e8-47ce60b2bc66","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:03:51.550Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"8a32faed-d33a-466a-84e8-47ce60b2bc66","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:03:51.550Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"8a32faed-d33a-466a-84e8-47ce60b2bc66","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:51.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83"}
{"level":"info","ts":"2026-06-11T10:03:51.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83"}
{"level":"info","ts":"2026-06-11T10:03:51.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:51.727Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:51.727Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:51.727Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:51.728Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.728Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.728Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:51.827Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83"}
{"level":"error","ts":"2026-06-11T10:03:51.827Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"0e3fa28c-a16b-477d-b534-0dd38d3a2e83","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.050Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","name":"kube-rbac-proxy-config-raycluster-auth","operation":"created"}
{"level":"error","ts":"2026-06-11T10:03:52.056Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.056Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa"}
{"level":"error","ts":"2026-06-11T10:03:52.056Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7e2780b5-f92c-4755-9950-c94d4146e4fa","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.056Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4d0133bb-4f95-4d76-a91c-779053d9fd5f","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:03:52.056Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4d0133bb-4f95-4d76-a91c-779053d9fd5f","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.056Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4d0133bb-4f95-4d76-a91c-779053d9fd5f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.069Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4d0133bb-4f95-4d76-a91c-779053d9fd5f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.069Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1fcdcc5e-af79-42b9-a421-389575614c70","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:03:52.069Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1fcdcc5e-af79-42b9-a421-389575614c70","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.069Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1fcdcc5e-af79-42b9-a421-389575614c70","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.069Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"972fdbd8-447b-4749-bac2-caf3f1df05f8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.070Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"10491c06-b3c6-4cd0-9202-243c633fa140","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.070Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"10491c06-b3c6-4cd0-9202-243c633fa140"}
{"level":"info","ts":"2026-06-11T10:03:52.134Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"972fdbd8-447b-4749-bac2-caf3f1df05f8","name":"raycluster-auth-head"}
{"level":"error","ts":"2026-06-11T10:03:52.135Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1fcdcc5e-af79-42b9-a421-389575614c70","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.136Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1fcdcc5e-af79-42b9-a421-389575614c70"}
{"level":"error","ts":"2026-06-11T10:03:52.136Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1fcdcc5e-af79-42b9-a421-389575614c70","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.141Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"659cd345-4be2-4c37-9867-032ee60d64f5","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:03:52.141Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"659cd345-4be2-4c37-9867-032ee60d64f5","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.141Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"659cd345-4be2-4c37-9867-032ee60d64f5","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d"}
{"level":"info","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d"}
{"level":"info","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:52.147Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:52.148Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:52.148Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d"}
{"level":"error","ts":"2026-06-11T10:03:52.148Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"f2061ae7-9784-420d-a7eb-90cca362127d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.149Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"972fdbd8-447b-4749-bac2-caf3f1df05f8","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:03:52.149Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"972fdbd8-447b-4749-bac2-caf3f1df05f8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.151Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"659cd345-4be2-4c37-9867-032ee60d64f5","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24"}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24"}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24"}
{"level":"error","ts":"2026-06-11T10:03:52.788Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"1a5354a2-9d09-4cdb-8457-820a1d328e24","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:03:54.069Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","enabled":true}
{"level":"info","ts":"2026-06-11T10:03:54.088Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","name":"raycluster-auth-head-jzs82"}
{"level":"info","ts":"2026-06-11T10:03:54.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:03:54.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.088Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:03:54.089Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405"}
{"level":"info","ts":"2026-06-11T10:03:54.089Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:03:54.089Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:03:54.089Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:03:54.089Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","enabled":true}
{"level":"info","ts":"2026-06-11T10:03:54.103Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","name":"raycluster-auth-small-group-worker-ffhq7"}
{"level":"info","ts":"2026-06-11T10:03:54.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.103Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-11T10:03:54.103Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"45cb4b3f-ccc0-4c72-abd3-e509e7289405","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","podName":"raycluster-auth-small-group-worker-ffhq7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","podName":"raycluster-auth-small-group-worker-ffhq7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-11T10:03:54.116Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:03:54.117Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"85635218-cdeb-4d5e-bfd6-f0bd0964ff03","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:54.117Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d8053632-0865-4d50-9b56-f3c4b7199177","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:54.117Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d8053632-0865-4d50-9b56-f3c4b7199177","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-vdznk","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e"}
{"level":"error","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"32813edd-1cc7-4d1c-af0d-0bfd1c03ed5e","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:03:54.123Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"70268a76-ff87-48b4-90f5-dc60ade65507","seconds":300}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86"}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86"}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.127Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:03:54.127Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"294fd94c-f365-4975-b183-98f373bbad86","seconds":300}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:03:54.128Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"155b6571-a53f-4608-bfbe-abbf92ea650a","seconds":300}
{"level":"info","ts":"2026-06-11T10:03:54.134Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"85635218-cdeb-4d5e-bfd6-f0bd0964ff03","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:03:54.139Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a2603835-59c4-46df-9422-74b915109e03","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:03:54.148Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"85635218-cdeb-4d5e-bfd6-f0bd0964ff03","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:03:54.148Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"85635218-cdeb-4d5e-bfd6-f0bd0964ff03","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.153Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:03:54.154Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:03:54.154Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"226f4e24-8d47-4a0d-9c11-9dadcb0d7792","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:54.154Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"52615228-48a5-4a68-b3b9-3e95bfb761b1","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:54.154Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"52615228-48a5-4a68-b3b9-3e95bfb761b1","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:03:54.160Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-vdznk","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-11T10:03:54.160Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e"}
{"level":"error","ts":"2026-06-11T10:03:54.160Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3004503a-5777-4ab7-bb80-942d8215900e","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:03:54.165Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"226f4e24-8d47-4a0d-9c11-9dadcb0d7792","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:03:54.166Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6c27d0eb-43e6-4e11-a03a-1534935fc3de","seconds":300}
{"level":"info","ts":"2026-06-11T10:03:54.175Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"226f4e24-8d47-4a0d-9c11-9dadcb0d7792","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:03:54.175Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"226f4e24-8d47-4a0d-9c11-9dadcb0d7792","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:03:54.544Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"fe1ccb92-5263-44d4-955c-cd65788bdcc1","seconds":300}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:03:54.553Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11bbff5d-3c64-4257-a160-b52a9ae5ed7d","seconds":300}
{"level":"info","ts":"2026-06-11T10:04:21.128Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e76ac1a8-f41e-4c08-8208-4d7ae16a6059","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:04:21.128Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e76ac1a8-f41e-4c08-8208-4d7ae16a6059","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:05:21.129Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe6ca61e-3372-44b9-b5de-c56343eec91d","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:05:21.130Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe6ca61e-3372-44b9-b5de-c56343eec91d","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:06:21.130Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"a14227d4-9d1e-4e66-a5f9-f326c909c778","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:06:21.131Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"a14227d4-9d1e-4e66-a5f9-f326c909c778","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:07:21.131Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"10604260-e172-42f8-a1c8-2c1e9a187f5a","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:07:21.131Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"10604260-e172-42f8-a1c8-2c1e9a187f5a","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:07:53.509Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a"}
{"level":"info","ts":"2026-06-11T10:07:53.509Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a"}
{"level":"info","ts":"2026-06-11T10:07:53.509Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"newHead":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-11T10:07:53.510Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"bb13670b-30c2-41a0-91fc-1b0288bf631a","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:03:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:07:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:07:53.523Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7683a81b-4f44-47f5-864a-8d3b39b73167","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:07:53.523Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"6247f5d9-f882-4cba-9a67-20a7288ec44b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:07:53.523Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7683a81b-4f44-47f5-864a-8d3b39b73167","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:07:53.558Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf"}
{"level":"info","ts":"2026-06-11T10:07:53.558Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf"}
{"level":"info","ts":"2026-06-11T10:07:53.558Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:07:53.559Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ea5270b0-1921-4dc6-aae1-e898d23948cf","seconds":300}
{"level":"info","ts":"2026-06-11T10:07:53.571Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"6247f5d9-f882-4cba-9a67-20a7288ec44b","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:07:53.592Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7683a81b-4f44-47f5-864a-8d3b39b73167","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-vdznk.svc","raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"],"ipAddresses":["10.133.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:07:53.596Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"6247f5d9-f882-4cba-9a67-20a7288ec44b","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:07:53.596Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"6247f5d9-f882-4cba-9a67-20a7288ec44b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:07:53.628Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7683a81b-4f44-47f5-864a-8d3b39b73167","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-vdznk.svc","raycluster-auth-worker-svc.test-ns-vdznk.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-vdznk.svc","*.raycluster-auth-worker-svc.test-ns-vdznk.svc.cluster.local","*-worker-*.test-ns-vdznk.svc","*-worker-*.test-ns-vdznk.svc.cluster.local"],"ipAddresses":["10.133.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:07:53.628Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7683a81b-4f44-47f5-864a-8d3b39b73167","rayCluster":"raycluster-auth","podIPs":["10.133.0.17"]}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:07:55.524Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"76373756-b450-4e98-9662-35c64efe52a7","seconds":300}
{"level":"info","ts":"2026-06-11T10:07:58.519Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04"}
{"level":"info","ts":"2026-06-11T10:07:58.519Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04"}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:07:58.520Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"426b1a37-ad3c-4b7d-979d-63b795fabf04","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:03.590Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9708ec5-dfaa-42b8-a8c5-4840da6a06c1","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:08:04.598Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ecab4481-8728-4d44-98c5-5745b8e84807","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:07:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:04.612Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"111c2dff-2a9f-47b1-8407-77f3fa2e13df","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:04.612Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d8bcfbb6-97db-45ba-8575-fd26ebed8e58","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:04.612Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"111c2dff-2a9f-47b1-8407-77f3fa2e13df","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:08:04.612Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"111c2dff-2a9f-47b1-8407-77f3fa2e13df","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-11T10:08:04.625Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d8bcfbb6-97db-45ba-8575-fd26ebed8e58","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:08:04.626Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"111c2dff-2a9f-47b1-8407-77f3fa2e13df","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-vdznk.svc","raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"],"ipAddresses":["10.133.0.17","10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:08:04.638Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d8bcfbb6-97db-45ba-8575-fd26ebed8e58","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:08:04.638Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d8bcfbb6-97db-45ba-8575-fd26ebed8e58","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:04.638Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"111c2dff-2a9f-47b1-8407-77f3fa2e13df","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-vdznk.svc","raycluster-auth-worker-svc.test-ns-vdznk.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-vdznk.svc","*.raycluster-auth-worker-svc.test-ns-vdznk.svc.cluster.local","*-worker-*.test-ns-vdznk.svc","*-worker-*.test-ns-vdznk.svc.cluster.local"],"ipAddresses":["10.133.0.17","10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:08:04.638Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"111c2dff-2a9f-47b1-8407-77f3fa2e13df","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:06.613Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6f5e7e7f-669b-49c1-a905-6eda95b2252a","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:08.572Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33"}
{"level":"info","ts":"2026-06-11T10:08:08.572Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33"}
{"level":"info","ts":"2026-06-11T10:08:08.572Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","podName":"raycluster-auth-head-jzs82","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:03:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","podName":"raycluster-auth-head-jzs82","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:03:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:08:08.573Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ef91bfb3-379a-461c-9a15-1a48d471da33","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:08.585Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b85306e3-fb64-4ddb-900f-60a8c5090638","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:08.585Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e6501ffd-e59c-4c09-9153-57940b04b125","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:08.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b85306e3-fb64-4ddb-900f-60a8c5090638","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:08:08.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b85306e3-fb64-4ddb-900f-60a8c5090638","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-11T10:08:08.585Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b85306e3-fb64-4ddb-900f-60a8c5090638","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-06-11T10:08:08.604Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e6501ffd-e59c-4c09-9153-57940b04b125","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:08:08.615Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e6501ffd-e59c-4c09-9153-57940b04b125","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:08:08.615Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e6501ffd-e59c-4c09-9153-57940b04b125","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51"}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51"}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","podName":"raycluster-auth-small-group-worker-ffhq7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:03:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","podName":"raycluster-auth-small-group-worker-ffhq7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:03:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:08:09.598Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"15d901c0-6596-4d6d-8750-129403324f51","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:09.611Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"529d9ab1-07b4-4e2e-a161-49d647484205","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:09.611Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fd41938e-1e16-4266-a7bc-960864e16803","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:09.611Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fd41938e-1e16-4266-a7bc-960864e16803","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:08:09.611Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fd41938e-1e16-4266-a7bc-960864e16803","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-11T10:08:09.611Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fd41938e-1e16-4266-a7bc-960864e16803","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-06-11T10:08:09.623Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"529d9ab1-07b4-4e2e-a161-49d647484205","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:08:09.638Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"529d9ab1-07b4-4e2e-a161-49d647484205","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:08:09.638Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"529d9ab1-07b4-4e2e-a161-49d647484205","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420"}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420"}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","podName":"raycluster-auth-small-group-worker-ffhq7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:03:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","podName":"raycluster-auth-small-group-worker-ffhq7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:03:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:10.587Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"5e7bc505-a141-4e06-8d2c-b099d9cc0420","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8"}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8"}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:08:15.623Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"eafd46da-c8a9-427c-b2cb-55a2027fe2c8","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:08:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:15.635Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c81cfa5e-036e-488c-aefc-d777b67b25d0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:15.635Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"06c1912c-8c5a-4d36-a801-5dcec40e0b04","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:15.635Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c81cfa5e-036e-488c-aefc-d777b67b25d0","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-11T10:08:15.635Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c81cfa5e-036e-488c-aefc-d777b67b25d0","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:08:15.635Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c81cfa5e-036e-488c-aefc-d777b67b25d0","rayCluster":"raycluster-auth","podIPs":["10.134.0.24","10.133.0.17"]}
{"level":"info","ts":"2026-06-11T10:08:15.650Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"06c1912c-8c5a-4d36-a801-5dcec40e0b04","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-11T10:08:15.664Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"06c1912c-8c5a-4d36-a801-5dcec40e0b04","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-11T10:08:15.664Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"06c1912c-8c5a-4d36-a801-5dcec40e0b04","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:16.183Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.183Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-5fsk2"}
{"level":"info","ts":"2026-06-11T10:08:16.193Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2a7b085f-c6f5-4747-a516-d706d35bc131","namespacedName":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:08:16.193Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251"}
{"level":"info","ts":"2026-06-11T10:08:16.193Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251"}
{"level":"info","ts":"2026-06-11T10:08:16.193Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.193Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.193Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"38a37553-d916-4b9a-b4eb-570a337d6463","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.193Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2a7b085f-c6f5-4747-a516-d706d35bc131","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.194Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2a7b085f-c6f5-4747-a516-d706d35bc131","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.194Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2a7b085f-c6f5-4747-a516-d706d35bc131","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.227Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","name":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster"}
{"level":"error","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:08:16.228Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:16.327Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251"}
{"level":"error","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"85530bed-0388-4462-8d70-80c7a1481251","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:16.328Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.427Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-5fsk2"}
{"level":"info","ts":"2026-06-11T10:08:16.427Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5fsk2","name":"raycluster"}}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"38a37553-d916-4b9a-b4eb-570a337d6463","cluster":"raycluster"}
{"level":"error","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7478ba2c-c029-45ee-8e04-32d6933a85ca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"594f1d1f-17b7-4d34-982d-5bf12b4e628d","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8"}
{"level":"error","ts":"2026-06-11T10:08:16.428Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e0df0a7-2cb4-4dd2-9302-4e64b8cb03c8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c"}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c"}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c"}
{"level":"error","ts":"2026-06-11T10:08:16.438Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"2e38427f-8f6d-4f8e-8941-4ab3befa500c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.440Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714"}
{"level":"info","ts":"2026-06-11T10:08:16.440Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:08:16.443Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.443Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-5fsk2"}
{"level":"info","ts":"2026-06-11T10:08:16.444Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"594f1d1f-17b7-4d34-982d-5bf12b4e628d","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3"}
{"level":"info","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3"}
{"level":"info","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.478Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.479Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.479Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3"}
{"level":"error","ts":"2026-06-11T10:08:16.479Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"827a367e-e21e-448f-8d05-832105a0cbc3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714"}
{"level":"error","ts":"2026-06-11T10:08:16.527Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2a7b085f-c6f5-4747-a516-d706d35bc131","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2a7b085f-c6f5-4747-a516-d706d35bc131"}
{"level":"error","ts":"2026-06-11T10:08:16.527Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2a7b085f-c6f5-4747-a516-d706d35bc131","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","namespacedName":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cab09f38-d9ae-4acb-a956-54d2061ca714"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e42e5d05-cb81-4202-8474-7fdc5ff097d9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.527Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e42e5d05-cb81-4202-8474-7fdc5ff097d9"}
{"level":"info","ts":"2026-06-11T10:08:16.534Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"594f1d1f-17b7-4d34-982d-5bf12b4e628d","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:08:16.534Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"594f1d1f-17b7-4d34-982d-5bf12b4e628d","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.535Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.535Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-5fsk2"}
{"level":"info","ts":"2026-06-11T10:08:16.546Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7e0f6e35-715f-49c0-9604-195cac709c4c","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.546Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6e9a2bd2-864d-45ec-b886-30417cf78548","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.546Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7e0f6e35-715f-49c0-9604-195cac709c4c"}
{"level":"info","ts":"2026-06-11T10:08:16.555Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","name":"raycluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6e9a2bd2-864d-45ec-b886-30417cf78548","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","name":"kuberay-gateway-access","namespace":"test-ns-5fsk2","operation":"created"}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd"}
{"level":"error","ts":"2026-06-11T10:08:16.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6754b663-0c64-49d2-a346-ea911e2027bd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.639Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","name":"test-ns-5fsk2-raycluster","operation":"created"}
{"level":"info","ts":"2026-06-11T10:08:16.640Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6e9a2bd2-864d-45ec-b886-30417cf78548","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:08:16.640Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6e9a2bd2-864d-45ec-b886-30417cf78548","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.645Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","name":"kube-rbac-proxy-config-raycluster","operation":"created"}
{"level":"error","ts":"2026-06-11T10:08:16.654Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.654Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef"}
{"level":"error","ts":"2026-06-11T10:08:16.654Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53f5274a-eed6-4d7e-aa4c-e56eedd9f0ef","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.654Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"240f3e81-76cf-4c10-a32e-f2c6ad928ab9","namespacedName":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:08:16.654Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"240f3e81-76cf-4c10-a32e-f2c6ad928ab9","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.654Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"240f3e81-76cf-4c10-a32e-f2c6ad928ab9","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.666Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c094e36f-18cf-4d60-a1ab-9bdf75388571","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.666Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cec1b184-30d6-4f5c-890d-2611820ed0a1","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.666Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cec1b184-30d6-4f5c-890d-2611820ed0a1"}
{"level":"info","ts":"2026-06-11T10:08:16.666Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"240f3e81-76cf-4c10-a32e-f2c6ad928ab9","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.666Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e9812e95-a930-4f49-88c0-978d0244e1f7","namespacedName":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:08:16.666Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e9812e95-a930-4f49-88c0-978d0244e1f7","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.666Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e9812e95-a930-4f49-88c0-978d0244e1f7","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.732Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c094e36f-18cf-4d60-a1ab-9bdf75388571","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:08:16.738Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e9812e95-a930-4f49-88c0-978d0244e1f7","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.745Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c094e36f-18cf-4d60-a1ab-9bdf75388571","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:08:16.745Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c094e36f-18cf-4d60-a1ab-9bdf75388571","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:16.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde"}
{"level":"info","ts":"2026-06-11T10:08:16.788Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde"}
{"level":"info","ts":"2026-06-11T10:08:16.788Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:16.788Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-11T10:08:16.788Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:16.788Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:16.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:16.789Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde"}
{"level":"error","ts":"2026-06-11T10:08:16.789Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e55b4790-e7cb-41fd-a606-c2ceff12edde","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:17.109Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a"}
{"level":"info","ts":"2026-06-11T10:08:17.109Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a"}
{"level":"info","ts":"2026-06-11T10:08:17.109Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","filter labels":["test-ns-5fsk2",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a"}
{"level":"error","ts":"2026-06-11T10:08:17.110Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"6fdf0cae-a9d4-447a-ade1-d776d747436a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:17.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd"}
{"level":"info","ts":"2026-06-11T10:08:17.635Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd"}
{"level":"info","ts":"2026-06-11T10:08:17.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:17.636Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4e8e6cf-36eb-46b4-a956-439285bfb8dd","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-5fsk2.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-5fsk2.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:17.751Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","enabled":true}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","name":"raycluster-head-t5q9r"}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99"}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:08:17.781Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","enabled":true}
{"level":"info","ts":"2026-06-11T10:08:17.800Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","name":"raycluster-small-group-worker-n98tf"}
{"level":"info","ts":"2026-06-11T10:08:17.801Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.801Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.801Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-11T10:08:17.801Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a6786ef-848d-4eec-889d-c36852f8fb99","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:17.816Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7dec41f3-86fb-4488-9a7d-16ff88c5c8cb","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:17.816Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"230aec05-6eaf-4d1d-ba74-5cddf6a8966e","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:17.816Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"230aec05-6eaf-4d1d-ba74-5cddf6a8966e"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.817Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.818Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:17.818Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5a9feeae-c421-420d-a74d-f6b9ddf28359","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"d6dee5f4-0199-4cc8-a776-6bd4004a8898","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:17.830Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7dec41f3-86fb-4488-9a7d-16ff88c5c8cb","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755"}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755"}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:17.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.838Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.838Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.838Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:08:17.838Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"283e7455-7cca-4fc9-af3c-6e364abed755","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:17.850Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7dec41f3-86fb-4488-9a7d-16ff88c5c8cb","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:08:17.850Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7dec41f3-86fb-4488-9a7d-16ff88c5c8cb","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:17.851Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2e290c52-e4d9-4d7e-8709-93b50cdfe164","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:17.851Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e5064a42-4769-462f-bdd9-7d1c662a72c6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:17.851Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e5064a42-4769-462f-bdd9-7d1c662a72c6"}
{"level":"info","ts":"2026-06-11T10:08:17.856Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:17.857Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c4c79b75-49e1-4731-9e38-1abfbc2c969e","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:17.865Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2e290c52-e4d9-4d7e-8709-93b50cdfe164","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:08:17.875Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2e290c52-e4d9-4d7e-8709-93b50cdfe164","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:08:17.875Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"2e290c52-e4d9-4d7e-8709-93b50cdfe164","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","podName":"raycluster-small-group-worker-n98tf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","podName":"raycluster-small-group-worker-n98tf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:18.253Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"7e53bbde-a6b6-44d4-a876-c8eb6385266a","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:18.279Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"8a858cab-e9b4-443d-929c-eb805cd3c59d","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:18.654Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d"}
{"level":"info","ts":"2026-06-11T10:08:18.654Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d"}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:18.655Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"9e562dff-7379-4358-9d6d-7274c3c7d42d","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb"}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb"}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:19.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:19.818Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:19.818Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:08:19.818Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:19.818Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"036b2fac-0902-474f-9beb-2b7d63105acb","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:21.132Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"75d0826b-37a8-4080-b385-77bb83ac16b6","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:21.132Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"75d0826b-37a8-4080-b385-77bb83ac16b6","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:08:21.132Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"75d0826b-37a8-4080-b385-77bb83ac16b6","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-11T10:08:21.132Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"75d0826b-37a8-4080-b385-77bb83ac16b6","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb"}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb"}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:21.179Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b41fe6a4-e352-4ea9-86ff-8a173d9f85fb","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe"}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe"}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:21.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.190Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:21.190Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"09dd320e-b716-4665-9567-9a4f9edb26fe","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab"}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab"}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:21.219Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.220Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:21.220Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"7cf88450-7357-4e51-b824-12c53e9515ab","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:21.244Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453"}
{"level":"info","ts":"2026-06-11T10:08:21.244Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453"}
{"level":"info","ts":"2026-06-11T10:08:21.244Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:21.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:21.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-ffhq7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:08:21.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:08:21.245Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:21.245Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"d4545133-d811-44f2-8afc-3d64be468453","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:21.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58"}
{"level":"info","ts":"2026-06-11T10:08:21.765Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58"}
{"level":"info","ts":"2026-06-11T10:08:21.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:21.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:08:21.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","worker Pod":"raycluster-auth-small-group-worker-ffhq7","shouldDelete":true,"reason":"The worker Pod raycluster-auth-small-group-worker-ffhq7 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-11T10:08:21.771Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:21.771Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-11T10:08:21.772Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:08:21.772Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:08:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:08:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:21.789Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58"}
{"level":"error","ts":"2026-06-11T10:08:21.789Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c3682184-20b7-48f0-b214-d22624cf6a58","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:21.789Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"8d045f5f-d128-4056-b859-612e507b0afc","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:21.789Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"9c1d66e0-b08a-4f48-968a-8c4f4e1302d8","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:21.789Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"9c1d66e0-b08a-4f48-968a-8c4f4e1302d8","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:08:21.789Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"9c1d66e0-b08a-4f48-968a-8c4f4e1302d8","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-11T10:08:21.789Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"9c1d66e0-b08a-4f48-968a-8c4f4e1302d8","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634"}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634"}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634","head Pod":"raycluster-auth-head-jzs82","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jzs82. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634","podName":"raycluster-auth-small-group-worker-ffhq7","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:21.794Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"11d357c3-926c-48fc-8ec3-c22c1f2a5634","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:21.798Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"8d045f5f-d128-4056-b859-612e507b0afc","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:21.881Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555"}
{"level":"info","ts":"2026-06-11T10:08:21.881Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555"}
{"level":"info","ts":"2026-06-11T10:08:21.882Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.882Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","Found 1 head Pod":"raycluster-auth-head-jzs82","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-11 10:07:58 +0000 UTC,FinishedAt:2026-06-11 10:08:21 +0000 UTC,ContainerID:cri-o://e970db9d4da804ef16c76407035f36097ab45b36f712f89d2de33f1ed341a05d,}"}
{"level":"info","ts":"2026-06-11T10:08:21.882Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","head Pod":"raycluster-auth-head-jzs82","shouldDelete":true,"reason":"The head Pod raycluster-auth-head-jzs82 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-11T10:08:21.891Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-auth-head-jzs82 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:21.891Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","error":"The head Pod raycluster-auth-head-jzs82 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-11T10:08:21.892Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:21Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:08:21.892Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:08:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:08:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:08:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:21Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:21.911Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"69215498-2971-42cb-8a72-03a91015176a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:21.911Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"264bddcb-8050-4244-a046-ce8068a1a8c4","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:21.911Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555"}
{"level":"info","ts":"2026-06-11T10:08:21.911Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"264bddcb-8050-4244-a046-ce8068a1a8c4","pod":"raycluster-auth-head-jzs82","ip":"10.133.0.17"}
{"level":"info","ts":"2026-06-11T10:08:21.911Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"264bddcb-8050-4244-a046-ce8068a1a8c4","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"error","ts":"2026-06-11T10:08:21.911Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"9b905dc1-e1ab-4528-a21d-d2cb493e2555","error":"The head Pod raycluster-auth-head-jzs82 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:21.911Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"264bddcb-8050-4244-a046-ce8068a1a8c4","rayCluster":"raycluster-auth","podIPs":["10.133.0.17","10.134.0.24"]}
{"level":"info","ts":"2026-06-11T10:08:21.916Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"69215498-2971-42cb-8a72-03a91015176a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:21.917Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576"}
{"level":"info","ts":"2026-06-11T10:08:21.917Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576"}
{"level":"info","ts":"2026-06-11T10:08:21.917Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:21.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-11T10:08:21.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-11T10:08:21.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:08:21.927Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:21.927Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6a5b45f5-1f7a-4763-94ec-26a63ecd3576","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f"}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f"}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f","podName":"raycluster-auth-head-jzs82","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:08:22.625Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"e314bda3-03d1-4a7d-8956-b4c03a54b29f","seconds":300}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.631Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:22.634Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.634Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:22.634Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.635Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.635Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.635Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"newHead":{"serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-11T10:08:22.635Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:08:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.17","serviceIP":"10.133.0.17","podName":"raycluster-auth-head-jzs82","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:21Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:08:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:08:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:03:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:03:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:08:22Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:08:22.647Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb"}
{"level":"error","ts":"2026-06-11T10:08:22.647Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3039001b-bd4c-48a7-94c4-a5cf592753eb","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.647Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4c53b483-276c-4b26-8dad-26dd00067e73","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.647Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"f5ee145e-d374-4a38-a9f5-0806ef97054a","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.647Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4c53b483-276c-4b26-8dad-26dd00067e73","pod":"raycluster-auth-small-group-worker-ffhq7","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","enabled":true}
{"level":"info","ts":"2026-06-11T10:08:22.652Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"f5ee145e-d374-4a38-a9f5-0806ef97054a","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:22.655Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.655Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:22.655Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.655Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.655Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.655Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32"}
{"level":"error","ts":"2026-06-11T10:08:22.655Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a244e16e-fbd6-43ff-8f20-267ed6185a32","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.660Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4c53b483-276c-4b26-8dad-26dd00067e73","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-vdznk.svc","raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"],"ipAddresses":["10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.666Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:22.670Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.670Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:22.670Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.670Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.670Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.670Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030"}
{"level":"error","ts":"2026-06-11T10:08:22.670Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"b9f1a450-d111-4e8d-9113-abaabacfa030","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.673Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","enabled":true}
{"level":"info","ts":"2026-06-11T10:08:22.676Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4c53b483-276c-4b26-8dad-26dd00067e73","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-vdznk.svc","raycluster-auth-worker-svc.test-ns-vdznk.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-vdznk.svc","*.raycluster-auth-worker-svc.test-ns-vdznk.svc.cluster.local","*-worker-*.test-ns-vdznk.svc","*-worker-*.test-ns-vdznk.svc.cluster.local"],"ipAddresses":["10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:08:22.676Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"4c53b483-276c-4b26-8dad-26dd00067e73","rayCluster":"raycluster-auth","podIPs":["10.134.0.24"]}
{"level":"error","ts":"2026-06-11T10:08:22.727Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952"}
{"level":"error","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"6d8d79d7-da5d-4454-85b9-7ffc764cb952","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:22.728Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:22.729Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.729Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.729Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:22.732Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.732Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:22.732Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.732Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.732Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.732Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47"}
{"level":"error","ts":"2026-06-11T10:08:22.732Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"a61be406-1794-4ac9-be8d-22f9ad54ab47","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.769Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:22.773Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.773Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:22.773Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.773Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.773Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.773Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409"}
{"level":"error","ts":"2026-06-11T10:08:22.773Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"16b271e1-6403-486c-84c8-73b5b7429409","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:22.934Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:22.939Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:22.939Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:22.939Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.939Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.939Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:22.939Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017"}
{"level":"error","ts":"2026-06-11T10:08:22.939Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"092b0138-b4e9-4994-89b0-07b4e9ed8017","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:23.260Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:23.264Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:23.264Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:23.264Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:23.264Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:23.264Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:23.264Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be"}
{"level":"error","ts":"2026-06-11T10:08:23.264Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"66fbe721-d238-48f5-aeb4-b13616a9a9be","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:23.905Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:23.909Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:23.909Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:23.909Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:23.909Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:23.909Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:23.909Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff"}
{"level":"error","ts":"2026-06-11T10:08:23.909Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"3b3a745f-2525-4253-bb76-d95f5c4f81ff","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-vdznk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:08:25.190Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","enabled":true}
{"level":"error","ts":"2026-06-11T10:08:25.194Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:25.194Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:25.194Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:25.194Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:25.194Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:25.194Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb"}
{"level":"error","ts":"2026-06-11T10:08:25.194Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"62e8d3e7-82b8-4370-bd37-f333d30373eb","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:26.950Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"66f3e8ec-4eec-4aab-a44b-52843443c16e","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:26.950Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"66f3e8ec-4eec-4aab-a44b-52843443c16e","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:26.950Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"66f3e8ec-4eec-4aab-a44b-52843443c16e","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:26.950Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe"}
{"level":"info","ts":"2026-06-11T10:08:26.950Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe"}
{"level":"error","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileHeadService-fm","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe","serviceSelectors":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/identifier":"raycluster-auth-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe","filter labels":["test-ns-vdznk",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe","error":"unable to find head service. cluster name raycluster-auth, filter labels [test-ns-vdznk map[ray.io/cluster:raycluster-auth ray.io/identifier:raycluster-auth-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe"}
{"level":"error","ts":"2026-06-11T10:08:26.953Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"c9ef2d08-9e48-4be1-b1f7-3023cf20a7fe","error":"services \"raycluster-auth-head-svc\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:26.961Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"66f3e8ec-4eec-4aab-a44b-52843443c16e","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.316Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"12f6c4f0-fdb1-482c-ac97-20aaa00f74c3","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.316Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"12f6c4f0-fdb1-482c-ac97-20aaa00f74c3","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.316Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"12f6c4f0-fdb1-482c-ac97-20aaa00f74c3","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.332Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7b3f763a-b579-41b3-be33-b920f64c2fbf","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.332Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7b3f763a-b579-41b3-be33-b920f64c2fbf","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.332Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"db2bed8d-9063-4bc3-8092-f63f141a6eee","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.332Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"12f6c4f0-fdb1-482c-ac97-20aaa00f74c3","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.332Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"12f6c4f0-fdb1-482c-ac97-20aaa00f74c3"}
{"level":"error","ts":"2026-06-11T10:08:27.332Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"12f6c4f0-fdb1-482c-ac97-20aaa00f74c3","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.335Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7b3f763a-b579-41b3-be33-b920f64c2fbf","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.335Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7b3f763a-b579-41b3-be33-b920f64c2fbf","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.335Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7b3f763a-b579-41b3-be33-b920f64c2fbf"}
{"level":"error","ts":"2026-06-11T10:08:27.335Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"7b3f763a-b579-41b3-be33-b920f64c2fbf","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.337Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"db2bed8d-9063-4bc3-8092-f63f141a6eee","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.338Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"35cd7f51-079d-4632-b2ad-3bd2f2516193","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.338Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"35cd7f51-079d-4632-b2ad-3bd2f2516193","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.338Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"35cd7f51-079d-4632-b2ad-3bd2f2516193","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.340Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0a716ab9-c1f3-427c-9f13-b2187ec72f8d","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.340Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0a716ab9-c1f3-427c-9f13-b2187ec72f8d","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.343Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0a716ab9-c1f3-427c-9f13-b2187ec72f8d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.343Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0a716ab9-c1f3-427c-9f13-b2187ec72f8d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.343Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0a716ab9-c1f3-427c-9f13-b2187ec72f8d"}
{"level":"error","ts":"2026-06-11T10:08:27.343Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0a716ab9-c1f3-427c-9f13-b2187ec72f8d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.352Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"35cd7f51-079d-4632-b2ad-3bd2f2516193","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.352Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"35cd7f51-079d-4632-b2ad-3bd2f2516193"}
{"level":"error","ts":"2026-06-11T10:08:27.352Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"35cd7f51-079d-4632-b2ad-3bd2f2516193","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.354Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"300d1c25-7da1-41c8-9796-41d858d71009","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.354Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"300d1c25-7da1-41c8-9796-41d858d71009","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.356Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"300d1c25-7da1-41c8-9796-41d858d71009","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.356Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"300d1c25-7da1-41c8-9796-41d858d71009","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.356Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"300d1c25-7da1-41c8-9796-41d858d71009"}
{"level":"error","ts":"2026-06-11T10:08:27.356Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"300d1c25-7da1-41c8-9796-41d858d71009","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.362Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"45f96789-2362-4ff0-95c4-b99953d4ded7","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.362Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"45f96789-2362-4ff0-95c4-b99953d4ded7","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.362Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"45f96789-2362-4ff0-95c4-b99953d4ded7","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.374Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"45f96789-2362-4ff0-95c4-b99953d4ded7","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.374Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"45f96789-2362-4ff0-95c4-b99953d4ded7"}
{"level":"error","ts":"2026-06-11T10:08:27.374Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"45f96789-2362-4ff0-95c4-b99953d4ded7","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.377Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d7644709-8773-495e-823d-1bff01711526","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.377Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d7644709-8773-495e-823d-1bff01711526","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.380Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d7644709-8773-495e-823d-1bff01711526","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.380Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d7644709-8773-495e-823d-1bff01711526","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.380Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d7644709-8773-495e-823d-1bff01711526"}
{"level":"error","ts":"2026-06-11T10:08:27.380Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d7644709-8773-495e-823d-1bff01711526","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.395Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"2f9821ed-c7d5-40f1-b7de-d4f6893318cf","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.395Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"2f9821ed-c7d5-40f1-b7de-d4f6893318cf","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.395Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"2f9821ed-c7d5-40f1-b7de-d4f6893318cf","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.408Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"2f9821ed-c7d5-40f1-b7de-d4f6893318cf","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.408Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"2f9821ed-c7d5-40f1-b7de-d4f6893318cf"}
{"level":"error","ts":"2026-06-11T10:08:27.408Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"2f9821ed-c7d5-40f1-b7de-d4f6893318cf","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.421Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"11f90e0d-9bbc-4187-91f5-54345a511e04","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.421Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"11f90e0d-9bbc-4187-91f5-54345a511e04","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.423Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"11f90e0d-9bbc-4187-91f5-54345a511e04","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.423Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"11f90e0d-9bbc-4187-91f5-54345a511e04","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.423Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"11f90e0d-9bbc-4187-91f5-54345a511e04"}
{"level":"error","ts":"2026-06-11T10:08:27.423Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"11f90e0d-9bbc-4187-91f5-54345a511e04","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.449Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe0a099f-78f0-4972-ad97-2f7c3bed6b70","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.449Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe0a099f-78f0-4972-ad97-2f7c3bed6b70","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.449Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe0a099f-78f0-4972-ad97-2f7c3bed6b70","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.460Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe0a099f-78f0-4972-ad97-2f7c3bed6b70","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.460Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe0a099f-78f0-4972-ad97-2f7c3bed6b70"}
{"level":"error","ts":"2026-06-11T10:08:27.460Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"fe0a099f-78f0-4972-ad97-2f7c3bed6b70","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.487Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"783a934d-752b-481c-86b5-56547bd623ba","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.492Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"783a934d-752b-481c-86b5-56547bd623ba","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.492Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"f2bf722f-bf45-41eb-a633-711feabfd2ba","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.496Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"f2bf722f-bf45-41eb-a633-711feabfd2ba","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.505Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c6bda524-3dec-4112-9dcd-4a563716d9b7","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.505Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c6bda524-3dec-4112-9dcd-4a563716d9b7","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.507Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c6bda524-3dec-4112-9dcd-4a563716d9b7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.507Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c6bda524-3dec-4112-9dcd-4a563716d9b7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.507Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c6bda524-3dec-4112-9dcd-4a563716d9b7"}
{"level":"error","ts":"2026-06-11T10:08:27.507Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c6bda524-3dec-4112-9dcd-4a563716d9b7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.541Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a7f7467-b4ff-4bb4-8930-e200e1db6847","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.541Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a7f7467-b4ff-4bb4-8930-e200e1db6847","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.541Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a7f7467-b4ff-4bb4-8930-e200e1db6847","cluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.554Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a7f7467-b4ff-4bb4-8930-e200e1db6847","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.554Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a7f7467-b4ff-4bb4-8930-e200e1db6847"}
{"level":"error","ts":"2026-06-11T10:08:27.554Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"3a7f7467-b4ff-4bb4-8930-e200e1db6847","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.668Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b2cefed9-def8-4ac3-b807-cb2be3fdaff2","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.668Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b2cefed9-def8-4ac3-b807-cb2be3fdaff2","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.671Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b2cefed9-def8-4ac3-b807-cb2be3fdaff2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.671Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b2cefed9-def8-4ac3-b807-cb2be3fdaff2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.671Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b2cefed9-def8-4ac3-b807-cb2be3fdaff2"}
{"level":"error","ts":"2026-06-11T10:08:27.671Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b2cefed9-def8-4ac3-b807-cb2be3fdaff2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.714Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c4144f2c-0ae9-477c-80a3-b36b0a722028","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.714Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c4144f2c-0ae9-477c-80a3-b36b0a722028","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.714Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c4144f2c-0ae9-477c-80a3-b36b0a722028","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d3b69883-f87a-424b-966e-01330886a14d","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d3b69883-f87a-424b-966e-01330886a14d","rayCluster":"raycluster-auth"}
{"level":"error","ts":"2026-06-11T10:08:27.729Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c4144f2c-0ae9-477c-80a3-b36b0a722028","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"reconcileID":"ae236ae4-0341-434a-9ddb-fd79850cddd1"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c4144f2c-0ae9-477c-80a3-b36b0a722028"}
{"level":"error","ts":"2026-06-11T10:08:27.729Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"c4144f2c-0ae9-477c-80a3-b36b0a722028","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"raycluster-auth-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"b25f071a-a43c-4fe5-815c-67a15dcc1ca4"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1daa7232-3ad2-47f1-aa33-2976dc7538d6","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1daa7232-3ad2-47f1-aa33-2976dc7538d6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1daa7232-3ad2-47f1-aa33-2976dc7538d6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1daa7232-3ad2-47f1-aa33-2976dc7538d6","namespace":"test-ns-vdznk","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:08:27.729Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1daa7232-3ad2-47f1-aa33-2976dc7538d6","name":"kuberay-gateway-access"}
{"level":"error","ts":"2026-06-11T10:08:27.732Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d3b69883-f87a-424b-966e-01330886a14d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:08:27.732Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d3b69883-f87a-424b-966e-01330886a14d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.732Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d3b69883-f87a-424b-966e-01330886a14d"}
{"level":"error","ts":"2026-06-11T10:08:27.732Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"d3b69883-f87a-424b-966e-01330886a14d","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-raycluster-auth\" is forbidden: unable to create new content in namespace test-ns-vdznk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:08:27.735Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1daa7232-3ad2-47f1-aa33-2976dc7538d6","name":"test-ns-vdznk-raycluster-auth","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:08:27.742Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.742Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-vdznk"}
{"level":"info","ts":"2026-06-11T10:08:27.752Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"1daa7232-3ad2-47f1-aa33-2976dc7538d6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.752Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e89aea20-1906-4df9-b9f1-5fbfff483378","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:27.752Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e89aea20-1906-4df9-b9f1-5fbfff483378"}
{"level":"info","ts":"2026-06-11T10:08:27.752Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"ab4f35c8-a5dc-4c67-b6fb-5586aba77627"}
{"level":"info","ts":"2026-06-11T10:08:27.752Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e89aea20-1906-4df9-b9f1-5fbfff483378","namespace":"test-ns-vdznk","authClusters":0}
{"level":"info","ts":"2026-06-11T10:08:27.752Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e8edec9d-9b27-4202-a38d-ac8a84c11d16","namespace":"test-ns-vdznk","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.752Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e89aea20-1906-4df9-b9f1-5fbfff483378","namespace":"test-ns-vdznk"}
{"level":"info","ts":"2026-06-11T10:08:27.753Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"e8edec9d-9b27-4202-a38d-ac8a84c11d16","namespace":"test-ns-vdznk","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.991Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"716439fd-9156-44f6-a3ca-bec6c1c3978f","namespace":"test-ns-vdznk","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:27.991Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"716439fd-9156-44f6-a3ca-bec6c1c3978f","namespace":"test-ns-vdznk","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-11T10:08:28.050Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0767e9db-e3da-45ed-ab09-a637c28e11c7","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-vdznk"}}
{"level":"info","ts":"2026-06-11T10:08:28.050Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0767e9db-e3da-45ed-ab09-a637c28e11c7"}
{"level":"info","ts":"2026-06-11T10:08:28.050Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0767e9db-e3da-45ed-ab09-a637c28e11c7","namespace":"test-ns-vdznk","authClusters":0}
{"level":"info","ts":"2026-06-11T10:08:28.050Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-vdznk"},"namespace":"test-ns-vdznk","name":"raycluster-auth","reconcileID":"0767e9db-e3da-45ed-ab09-a637c28e11c7","namespace":"test-ns-vdznk"}
{"level":"info","ts":"2026-06-11T10:08:46.528Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"4602bd12-8ab1-4d0c-ae38-9563683176e5","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:08:46.528Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"4602bd12-8ab1-4d0c-ae38-9563683176e5","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:08:46.546Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"4602bd12-8ab1-4d0c-ae38-9563683176e5","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-5fsk2.svc","raycluster-head-svc.test-ns-5fsk2.svc.cluster.local"],"ipAddresses":["10.134.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:08:46.577Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"4602bd12-8ab1-4d0c-ae38-9563683176e5","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-5fsk2.svc","raycluster-worker-svc.test-ns-5fsk2.svc.cluster.local","*.raycluster-worker-svc.test-ns-5fsk2.svc","*.raycluster-worker-svc.test-ns-5fsk2.svc.cluster.local","*-worker-*.test-ns-5fsk2.svc","*-worker-*.test-ns-5fsk2.svc.cluster.local"],"ipAddresses":["10.134.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:08:46.577Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"4602bd12-8ab1-4d0c-ae38-9563683176e5"}
{"level":"info","ts":"2026-06-11T10:09:16.577Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"dcc2cebd-357a-49a2-9fbc-879d721b4e81","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:09:16.577Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"dcc2cebd-357a-49a2-9fbc-879d721b4e81","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:09:16.577Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"dcc2cebd-357a-49a2-9fbc-879d721b4e81","rayCluster":"raycluster","podIPs":["10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:10:16.578Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e6b8bef3-59bb-42b2-8b9d-18470c048ac0","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:10:16.579Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e6b8bef3-59bb-42b2-8b9d-18470c048ac0","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:10:16.579Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e6b8bef3-59bb-42b2-8b9d-18470c048ac0","rayCluster":"raycluster","podIPs":["10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:11:16.579Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"29af6b3c-4dd1-4c48-a3ee-e32c933e4eb0","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:11:16.579Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"29af6b3c-4dd1-4c48-a3ee-e32c933e4eb0","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:11:16.579Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"29af6b3c-4dd1-4c48-a3ee-e32c933e4eb0","rayCluster":"raycluster","podIPs":["10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"newHead":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-11T10:11:53.381Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"12dac513-0e8c-40b0-a9ca-b43609181f60","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:08:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:11:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:11:53.397Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5ee21f93-8c93-4470-859b-052d2b3bd515","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:11:53.397Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"50869bdc-1120-4af9-aa08-ef6fb45bbbd7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:11:53.397Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"50869bdc-1120-4af9-aa08-ef6fb45bbbd7","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:11:53.397Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"50869bdc-1120-4af9-aa08-ef6fb45bbbd7","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:11:53.420Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5ee21f93-8c93-4470-859b-052d2b3bd515","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:11:53.433Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"50869bdc-1120-4af9-aa08-ef6fb45bbbd7","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-5fsk2.svc","raycluster-head-svc.test-ns-5fsk2.svc.cluster.local"],"ipAddresses":["10.132.0.27","10.134.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:11:53.450Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5ee21f93-8c93-4470-859b-052d2b3bd515","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:11:53.450Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5ee21f93-8c93-4470-859b-052d2b3bd515","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:11:53.484Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"50869bdc-1120-4af9-aa08-ef6fb45bbbd7","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-5fsk2.svc","raycluster-worker-svc.test-ns-5fsk2.svc.cluster.local","*.raycluster-worker-svc.test-ns-5fsk2.svc","*.raycluster-worker-svc.test-ns-5fsk2.svc.cluster.local","*-worker-*.test-ns-5fsk2.svc","*-worker-*.test-ns-5fsk2.svc.cluster.local"],"ipAddresses":["10.132.0.27","10.134.0.25","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:11:53.484Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"50869bdc-1120-4af9-aa08-ef6fb45bbbd7","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:11:55.397Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"83eaa860-2533-49d5-bf96-68895a6f6040","seconds":300}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66"}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66"}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:11:58.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:11:58.411Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:11:58.411Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:11:58.411Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:11:58.411Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"c76b6f86-8bb3-4393-a0d0-24176666ce66","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","podName":"raycluster-small-group-worker-n98tf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","podName":"raycluster-small-group-worker-n98tf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:03.576Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"ed7d2031-40a6-49fb-a5ba-4503117dda0d","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21"}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21"}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:04.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:04.583Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:12:04.583Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"e908669b-a24b-49ae-a9d7-a26cc41a3d21","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:11:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:04.595Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"a6ce2c1f-8f92-4793-948b-f25f6653a974","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:04.595Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e94e4a44-1a1e-40ea-850e-17ed10616cfe","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:04.595Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e94e4a44-1a1e-40ea-850e-17ed10616cfe","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:12:04.595Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e94e4a44-1a1e-40ea-850e-17ed10616cfe","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:12:04.595Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"e94e4a44-1a1e-40ea-850e-17ed10616cfe","rayCluster":"raycluster","podIPs":["10.134.0.25","10.132.0.27"]}
{"level":"info","ts":"2026-06-11T10:12:04.608Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"a6ce2c1f-8f92-4793-948b-f25f6653a974","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:12:04.619Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"a6ce2c1f-8f92-4793-948b-f25f6653a974","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:12:04.619Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"a6ce2c1f-8f92-4793-948b-f25f6653a974","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1"}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1"}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:06.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:06.596Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:06.596Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","podName":"raycluster-head-t5q9r","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:06.596Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:06.596Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"786a3c4f-3f25-47a4-bd89-ea3afe19fca1","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37"}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37"}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:08.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","podName":"raycluster-head-t5q9r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:12:08.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","podName":"raycluster-head-t5q9r","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:12:08.445Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:12:08.445Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"b34bb04a-b1f7-49ac-bca5-8cbf28b2be37","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:08.456Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f9e17925-7cf5-44ec-9828-20170a51de1d","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:08.456Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"fb282047-02af-4838-a7ec-36e3a232603a","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:08.456Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f9e17925-7cf5-44ec-9828-20170a51de1d","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:12:08.456Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f9e17925-7cf5-44ec-9828-20170a51de1d","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:12:08.456Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f9e17925-7cf5-44ec-9828-20170a51de1d","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:12:08.475Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"fb282047-02af-4838-a7ec-36e3a232603a","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:12:08.484Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"fb282047-02af-4838-a7ec-36e3a232603a","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:12:08.484Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"fb282047-02af-4838-a7ec-36e3a232603a","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3"}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3"}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","podName":"raycluster-small-group-worker-n98tf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","podName":"raycluster-small-group-worker-n98tf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:12:09.480Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"5acc853b-4471-423a-aedc-f20d8fede0b3","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:09.495Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cc801482-afc5-4456-84e7-0781558c6bf1","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:09.495Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7afd64d5-0e15-4339-a573-c2af9e808c95","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:09.495Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7afd64d5-0e15-4339-a573-c2af9e808c95","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:12:09.495Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7afd64d5-0e15-4339-a573-c2af9e808c95","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:12:09.495Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"7afd64d5-0e15-4339-a573-c2af9e808c95","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:12:09.508Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cc801482-afc5-4456-84e7-0781558c6bf1","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:12:09.523Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cc801482-afc5-4456-84e7-0781558c6bf1","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:12:09.523Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cc801482-afc5-4456-84e7-0781558c6bf1","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c"}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c"}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","podName":"raycluster-small-group-worker-n98tf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","podName":"raycluster-small-group-worker-n98tf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:08:17Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:10.456Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"36fd159c-797b-4367-bb8b-16988282ce8c","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3"}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3"}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:15.607Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:15.608Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:12:15.608Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"02fe73c5-2555-4c51-994a-c384b8d467b3","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:15.620Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b88acd34-b114-45a2-8808-b9e4fb4001e2","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:15.620Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"ce3c4776-9884-4fe2-af7a-8644a03548d0","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:15.620Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b88acd34-b114-45a2-8808-b9e4fb4001e2","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:12:15.620Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b88acd34-b114-45a2-8808-b9e4fb4001e2","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:12:15.620Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b88acd34-b114-45a2-8808-b9e4fb4001e2","rayCluster":"raycluster","podIPs":["10.134.0.25","10.132.0.27"]}
{"level":"info","ts":"2026-06-11T10:12:15.633Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"ce3c4776-9884-4fe2-af7a-8644a03548d0","name":"raycluster-head"}
{"level":"info","ts":"2026-06-11T10:12:15.644Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"ce3c4776-9884-4fe2-af7a-8644a03548d0","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-11T10:12:15.644Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"ce3c4776-9884-4fe2-af7a-8644a03548d0","cluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:15.820Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"49a70d60-9cc2-4a20-827e-dcde111e57e6","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:15.820Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"49a70d60-9cc2-4a20-827e-dcde111e57e6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:12:15.831Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"49a70d60-9cc2-4a20-827e-dcde111e57e6"}
{"level":"info","ts":"2026-06-11T10:12:15.831Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"49a70d60-9cc2-4a20-827e-dcde111e57e6","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:12:15.831Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"49a70d60-9cc2-4a20-827e-dcde111e57e6","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:12:15.831Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"49a70d60-9cc2-4a20-827e-dcde111e57e6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:12:15.842Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5c39cd19-325e-4384-bd54-fede34821e62","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:15.842Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5c39cd19-325e-4384-bd54-fede34821e62","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.842Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5c39cd19-325e-4384-bd54-fede34821e62","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:15.842Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5c39cd19-325e-4384-bd54-fede34821e62","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:15.861Z","logger":"raycluster-resource","msg":"default","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.861Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-rpnf8","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"df417b6e-fda8-4ede-9347-2917ec535ebb","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","mode":"IntegratedOAuth","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.873Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5c39cd19-325e-4384-bd54-fede34821e62","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5c39cd19-325e-4384-bd54-fede34821e62"}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5c39cd19-325e-4384-bd54-fede34821e62","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"2eba79fc-cea9-44b8-a30f-7c0c44d3a337","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"2eba79fc-cea9-44b8-a30f-7c0c44d3a337","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"2eba79fc-cea9-44b8-a30f-7c0c44d3a337","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"2eba79fc-cea9-44b8-a30f-7c0c44d3a337"}
{"level":"info","ts":"2026-06-11T10:12:15.874Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"2eba79fc-cea9-44b8-a30f-7c0c44d3a337","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:15.882Z","logger":"raycluster-resource","msg":"default","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.882Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-rpnf8","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:15.890Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","name":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:15.890Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:15.890Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-rpnf8-head-svc"}}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","name":"counter-rpnf8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"df417b6e-fda8-4ede-9347-2917ec535ebb","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.890Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.927Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e44b55ae-7637-4e76-a4d3-1e7e178cc4f7","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"28ea81e9-0bb3-4baa-ba6d-24dd49ebc406","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:15.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"28ea81e9-0bb3-4baa-ba6d-24dd49ebc406","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"28ea81e9-0bb3-4baa-ba6d-24dd49ebc406","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:15.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"28ea81e9-0bb3-4baa-ba6d-24dd49ebc406"}
{"level":"info","ts":"2026-06-11T10:12:15.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"28ea81e9-0bb3-4baa-ba6d-24dd49ebc406","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","name":"counter-rpnf8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-rpnf8\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccphs","name":"counter-rpnf8"}}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952"}
{"level":"error","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"21170029-d692-4f46-9b0b-da743a4e2952","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879"}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879"}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-rpnf8-head-svc"}}
{"level":"info","ts":"2026-06-11T10:12:15.943Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","name":"counter-rpnf8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:15.948Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315"}
{"level":"info","ts":"2026-06-11T10:12:15.948Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:12:15.948Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.956Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","name":"counter-rpnf8-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:12:15.970Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e44b55ae-7637-4e76-a4d3-1e7e178cc4f7","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ae27c94c-f343-4c38-b9d1-d9a3af656d26","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ae27c94c-f343-4c38-b9d1-d9a3af656d26","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ae27c94c-f343-4c38-b9d1-d9a3af656d26","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ae27c94c-f343-4c38-b9d1-d9a3af656d26"}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ae27c94c-f343-4c38-b9d1-d9a3af656d26","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879"}
{"level":"error","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"31f6b195-97e6-42c6-8fd5-5336d37f0879","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf"}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf"}
{"level":"info","ts":"2026-06-11T10:12:15.971Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf"}
{"level":"error","ts":"2026-06-11T10:12:15.972Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd57a224-b9bd-4c48-a354-c9bc391d55bf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:15.977Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","name":"kuberay-gateway-access","namespace":"test-ns-ccphs","operation":"created"}
{"level":"info","ts":"2026-06-11T10:12:15.977Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:16.027Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","name":"test-ns-ccphs-counter-rpnf8","operation":"created"}
{"level":"info","ts":"2026-06-11T10:12:16.027Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.028Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e"}
{"level":"error","ts":"2026-06-11T10:12:16.028Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"cd2a4fa3-1234-4041-b9ce-6ef51fdbe39e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.028Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:12:16.037Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","name":"kube-rbac-proxy-config-counter-rpnf8","operation":"created"}
{"level":"info","ts":"2026-06-11T10:12:16.037Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e44b55ae-7637-4e76-a4d3-1e7e178cc4f7","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:16.037Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e44b55ae-7637-4e76-a4d3-1e7e178cc4f7","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.037Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e98392d3-ca3a-450a-b7a5-e35e362d33dd","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.127Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315"}
{"level":"info","ts":"2026-06-11T10:12:16.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed"}
{"level":"info","ts":"2026-06-11T10:12:16.127Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed"}
{"level":"info","ts":"2026-06-11T10:12:16.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:16.127Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","rayCluster":"counter-rpnf8","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:12:16.127Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-rpnf8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.127Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315"}
{"level":"error","ts":"2026-06-11T10:12:16.127Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"a04804d4-c2a5-4a6c-bd96-10aaf68b8315","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-rpnf8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5ec6e805-26c7-4419-bb5f-da03c25e3e4c","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.127Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:16.128Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-11T10:12:16.128Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rpnf8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865"}
{"level":"error","ts":"2026-06-11T10:12:16.128Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"805e409d-d87c-4b75-ab16-e91675add865","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rpnf8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"c8724aae-8d81-4c16-88fb-2746c10c09c0","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"error","ts":"2026-06-11T10:12:16.128Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"8eb42901-8aea-4958-a753-3fda7d0a96ed","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"c8724aae-8d81-4c16-88fb-2746c10c09c0","mode":"IntegratedOAuth","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.128Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"c8724aae-8d81-4c16-88fb-2746c10c09c0","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.132Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e98392d3-ca3a-450a-b7a5-e35e362d33dd","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:16.143Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"c8724aae-8d81-4c16-88fb-2746c10c09c0","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.143Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e34838a8-b0d3-4d0b-a7dc-98968f410ec7","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:16.143Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e34838a8-b0d3-4d0b-a7dc-98968f410ec7","mode":"IntegratedOAuth","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.143Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e34838a8-b0d3-4d0b-a7dc-98968f410ec7","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5ec6e805-26c7-4419-bb5f-da03c25e3e4c"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3"}
{"level":"error","ts":"2026-06-11T10:12:16.227Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5ec6e805-26c7-4419-bb5f-da03c25e3e4c","error":"certificates.cert-manager.io \"ray-worker-cert-counter-rpnf8\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:12:16.227Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5ec6e805-26c7-4419-bb5f-da03c25e3e4c","error":"certificates.cert-manager.io \"ray-worker-cert-counter-rpnf8\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5ec6e805-26c7-4419-bb5f-da03c25e3e4c"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:16.227Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e34838a8-b0d3-4d0b-a7dc-98968f410ec7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rpnf8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e34838a8-b0d3-4d0b-a7dc-98968f410ec7"}
{"level":"error","ts":"2026-06-11T10:12:16.227Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e34838a8-b0d3-4d0b-a7dc-98968f410ec7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-rpnf8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:12:16.227Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:16.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-11T10:12:16.227Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5ec6e805-26c7-4419-bb5f-da03c25e3e4c","error":"certificates.cert-manager.io \"ray-worker-cert-counter-rpnf8\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3"}
{"level":"error","ts":"2026-06-11T10:12:16.228Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9b8fa746-5d13-4a04-8f24-138e42e7f5c3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"4f07ef2e-62fc-4c12-9f84-d65829b43c6c","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e98392d3-ca3a-450a-b7a5-e35e362d33dd","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"e98392d3-ca3a-450a-b7a5-e35e362d33dd","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"4f07ef2e-62fc-4c12-9f84-d65829b43c6c"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"37097dbc-7cf2-4f47-8fef-66247eb59be0","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f9460355-2c81-444f-86f6-d120664515b6","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.228Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"37097dbc-7cf2-4f47-8fef-66247eb59be0"}
{"level":"info","ts":"2026-06-11T10:12:16.233Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"261936af-d7e3-4228-9456-067c5c6a261b","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:16.233Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"261936af-d7e3-4228-9456-067c5c6a261b","mode":"IntegratedOAuth","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.233Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"261936af-d7e3-4228-9456-067c5c6a261b","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.238Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"768be187-289c-48ca-921f-e4fce10823ea","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.238Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"768be187-289c-48ca-921f-e4fce10823ea"}
{"level":"info","ts":"2026-06-11T10:12:16.242Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f9460355-2c81-444f-86f6-d120664515b6","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:16.327Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"261936af-d7e3-4228-9456-067c5c6a261b","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.327Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c32b706b-95a6-4192-9558-45b96610e20c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:16.327Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c32b706b-95a6-4192-9558-45b96610e20c","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.327Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c32b706b-95a6-4192-9558-45b96610e20c","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:16.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c32b706b-95a6-4192-9558-45b96610e20c"}
{"level":"info","ts":"2026-06-11T10:12:16.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c32b706b-95a6-4192-9558-45b96610e20c","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:16.427Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568"}
{"level":"info","ts":"2026-06-11T10:12:16.427Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568"}
{"level":"info","ts":"2026-06-11T10:12:16.427Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f9460355-2c81-444f-86f6-d120664515b6","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:16.427Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:16.427Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f9460355-2c81-444f-86f6-d120664515b6","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:16.427Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:16.427Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.427Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:16.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.428Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568"}
{"level":"error","ts":"2026-06-11T10:12:16.428Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"d7edfc77-f317-4083-a9f1-bafdf1de9568","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.580Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"933766c5-0823-48c9-bcaf-150bd7b88907","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:12:16.580Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"933766c5-0823-48c9-bcaf-150bd7b88907","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:12:16.580Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"933766c5-0823-48c9-bcaf-150bd7b88907","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:12:16.580Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"933766c5-0823-48c9-bcaf-150bd7b88907","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54"}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54"}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54"}
{"level":"error","ts":"2026-06-11T10:12:16.749Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"09ea7f79-c286-41b6-8c43-7fb94b1abd54","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5"}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5"}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","error":"head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"error","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found"}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","filter labels":["test-ns-ccphs",{"ray.io/cluster":"counter-rpnf8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5"}
{"level":"error","ts":"2026-06-11T10:12:17.390Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7db8f199-3ec7-484d-a4a1-3abbbae1e5c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-rpnf8 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b"}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b"}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:17.621Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"dfe1a087-c33e-4148-a204-05a55329cd6b","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","mode":"IntegratedOAuth","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","cluster":"counter-rpnf8","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","cluster":"counter-rpnf8","authType":"IntegratedOAuth","serviceAccount":"counter-rpnf8-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-rpnf8","ray.io/group":"headgroup","ray.io/identifier":"counter-rpnf8-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local","num-cpus":"4","num-gpus":"4","resources":"'{\"R1\": 4}'"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-06-11T10:12:18.671Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","enabled":true}
{"level":"info","ts":"2026-06-11T10:12:18.691Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","name":"counter-rpnf8-head-m7x9j"}
{"level":"info","ts":"2026-06-11T10:12:18.691Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.691Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.691Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-rpnf8-head-svc"},"newHead":{"podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"}}
{"level":"info","ts":"2026-06-11T10:12:18.691Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"df927a7b-689e-4318-8a8b-d1143a150518","name":"counter-rpnf8","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"aa799cc6-8205-40d0-91be-e9cc55ca9f0c","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"71f0a72d-39c5-450d-bec4-caae8dd1add4","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ab98780d-ecdc-4add-a916-809b225738ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"71f0a72d-39c5-450d-bec4-caae8dd1add4","rayCluster":"counter-rpnf8","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ab98780d-ecdc-4add-a916-809b225738ee","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ab98780d-ecdc-4add-a916-809b225738ee","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ab98780d-ecdc-4add-a916-809b225738ee"}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"ab98780d-ecdc-4add-a916-809b225738ee","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:18.702Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4589dae7-27a8-47ad-80ee-c02cbb0d35ca","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e"}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e"}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:18.714Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0cc039c1-b7e9-47b7-a55e-2a518bc18a4e","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:18.717Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"aa799cc6-8205-40d0-91be-e9cc55ca9f0c","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:18.731Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"aa799cc6-8205-40d0-91be-e9cc55ca9f0c","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:18.731Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"aa799cc6-8205-40d0-91be-e9cc55ca9f0c","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211"}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211"}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:12:18.736Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"dc578f7e-63c4-452a-b86b-2e5d8547e211","name":"counter-rpnf8","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"3471aad9-b460-4ec1-a4b2-c445fe5fd2de","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5b3a81ad-da6f-4aa7-8b03-91dba29a7b71","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"3471aad9-b460-4ec1-a4b2-c445fe5fd2de","rayCluster":"counter-rpnf8","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5f1aaa4c-ce46-4db5-9eca-717175842d96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5f1aaa4c-ce46-4db5-9eca-717175842d96","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5f1aaa4c-ce46-4db5-9eca-717175842d96","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5f1aaa4c-ce46-4db5-9eca-717175842d96"}
{"level":"info","ts":"2026-06-11T10:12:18.749Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"5f1aaa4c-ce46-4db5-9eca-717175842d96","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:18.761Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5b3a81ad-da6f-4aa7-8b03-91dba29a7b71","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:18.776Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5b3a81ad-da6f-4aa7-8b03-91dba29a7b71","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:18.776Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"5b3a81ad-da6f-4aa7-8b03-91dba29a7b71","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.843Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"e349112a-a060-4e21-aac7-dda67a181ed4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:18.843Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"e349112a-a060-4e21-aac7-dda67a181ed4","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:18.843Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"e349112a-a060-4e21-aac7-dda67a181ed4","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:18.843Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"e349112a-a060-4e21-aac7-dda67a181ed4"}
{"level":"info","ts":"2026-06-11T10:12:18.843Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"e349112a-a060-4e21-aac7-dda67a181ed4","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852"}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852"}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:19.146Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"50018810-2539-41b4-9931-add1df822852","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258"}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258"}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"newHead":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"}}
{"level":"info","ts":"2026-06-11T10:12:19.488Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"7b0b8503-0604-4bc6-a309-e0830fde0258","name":"counter-rpnf8","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"7a98b177-cd00-4d5d-abad-bbf424d1e7db","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0a9b9172-3156-43d4-bea3-dc2fc0792b73","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"7a98b177-cd00-4d5d-abad-bbf424d1e7db","pod":"counter-rpnf8-head-m7x9j","ip":"10.133.0.18"}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"706fd28d-c2f8-43e1-bbee-bc1a17edeb12","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"706fd28d-c2f8-43e1-bbee-bc1a17edeb12","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"706fd28d-c2f8-43e1-bbee-bc1a17edeb12","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"706fd28d-c2f8-43e1-bbee-bc1a17edeb12"}
{"level":"info","ts":"2026-06-11T10:12:19.500Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"706fd28d-c2f8-43e1-bbee-bc1a17edeb12","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:19.513Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"7a98b177-cd00-4d5d-abad-bbf424d1e7db","certificate":"ray-head-cert-counter-rpnf8","dnsNames":["counter-rpnf8-head-svc","localhost","counter-rpnf8-head-svc.test-ns-ccphs.svc","counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:12:19.514Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0a9b9172-3156-43d4-bea3-dc2fc0792b73","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:19.525Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"7a98b177-cd00-4d5d-abad-bbf424d1e7db","certificate":"ray-worker-cert-counter-rpnf8","dnsNames":["counter-rpnf8-worker-svc","localhost","counter-rpnf8-worker-svc.test-ns-ccphs.svc","counter-rpnf8-worker-svc.test-ns-ccphs.svc.cluster.local","*.counter-rpnf8-worker-svc.test-ns-ccphs.svc","*.counter-rpnf8-worker-svc.test-ns-ccphs.svc.cluster.local","*-worker-*.test-ns-ccphs.svc","*-worker-*.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.18","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:12:19.525Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"7a98b177-cd00-4d5d-abad-bbf424d1e7db","rayCluster":"counter-rpnf8","podIPs":["10.133.0.18"]}
{"level":"info","ts":"2026-06-11T10:12:19.529Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0a9b9172-3156-43d4-bea3-dc2fc0792b73","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:19.529Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0a9b9172-3156-43d4-bea3-dc2fc0792b73","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d"}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d"}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:20.703Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"e6fea467-a500-4c14-bc6c-ced86dc55f5d","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:21.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"aa7b6439-95d7-4797-8e5c-38829a1d4376","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:21.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"aa7b6439-95d7-4797-8e5c-38829a1d4376","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:21.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"aa7b6439-95d7-4797-8e5c-38829a1d4376","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:21.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"aa7b6439-95d7-4797-8e5c-38829a1d4376"}
{"level":"info","ts":"2026-06-11T10:12:21.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"aa7b6439-95d7-4797-8e5c-38829a1d4376","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e"}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e"}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e","podName":"counter-rpnf8-head-m7x9j","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:24.508Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba73449a-ecc2-4e07-a7aa-b80b43447b4e","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:24.845Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c2e976f5-9086-4c55-8551-09dc5a04ed71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:24.845Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c2e976f5-9086-4c55-8551-09dc5a04ed71","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:24.845Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c2e976f5-9086-4c55-8551-09dc5a04ed71","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:24.845Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c2e976f5-9086-4c55-8551-09dc5a04ed71"}
{"level":"info","ts":"2026-06-11T10:12:24.845Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c2e976f5-9086-4c55-8551-09dc5a04ed71","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:25.513Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b"}
{"level":"info","ts":"2026-06-11T10:12:25.513Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b"}
{"level":"info","ts":"2026-06-11T10:12:25.513Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:25.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:25.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:25.513Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b","podName":"counter-rpnf8-head-m7x9j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:12:25.514Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b","podName":"counter-rpnf8-head-m7x9j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:12:25.514Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:12:25.514Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"ba37300f-9d7d-432c-9a95-8da14ff2937b","name":"counter-rpnf8","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:25.526Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"98a7a29c-abb4-4025-97ae-d904d11788d0","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:25.526Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"eac0725b-a405-4bf8-a65b-35ff3e5573f6","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:25.526Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"98a7a29c-abb4-4025-97ae-d904d11788d0","pod":"counter-rpnf8-head-m7x9j","ip":"10.133.0.18"}
{"level":"info","ts":"2026-06-11T10:12:25.526Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"98a7a29c-abb4-4025-97ae-d904d11788d0","rayCluster":"counter-rpnf8","podIPs":["10.133.0.18"]}
{"level":"info","ts":"2026-06-11T10:12:25.526Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"3ed41860-c0e9-4504-9576-c3a677a655d1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:25.527Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"3ed41860-c0e9-4504-9576-c3a677a655d1","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:25.527Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"3ed41860-c0e9-4504-9576-c3a677a655d1","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:25.527Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"3ed41860-c0e9-4504-9576-c3a677a655d1"}
{"level":"info","ts":"2026-06-11T10:12:25.527Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"3ed41860-c0e9-4504-9576-c3a677a655d1","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:25.538Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"eac0725b-a405-4bf8-a65b-35ff3e5573f6","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:25.549Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"eac0725b-a405-4bf8-a65b-35ff3e5573f6","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:25.549Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"eac0725b-a405-4bf8-a65b-35ff3e5573f6","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1"}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1"}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1","podName":"counter-rpnf8-head-m7x9j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1","podName":"counter-rpnf8-head-m7x9j","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:27.527Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"0dd63eac-58ed-4054-a451-4a7306e9dcc1","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:27.846Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c9928f95-c587-4f8f-b128-2b0d9539c353","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:27.846Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c9928f95-c587-4f8f-b128-2b0d9539c353","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:27.846Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c9928f95-c587-4f8f-b128-2b0d9539c353","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:27.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c9928f95-c587-4f8f-b128-2b0d9539c353"}
{"level":"info","ts":"2026-06-11T10:12:27.847Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"c9928f95-c587-4f8f-b128-2b0d9539c353","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:30.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"00181db5-aea4-4875-b5f7-c91ef8f20487","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:30.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"00181db5-aea4-4875-b5f7-c91ef8f20487","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:30.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"00181db5-aea4-4875-b5f7-c91ef8f20487","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:30.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"00181db5-aea4-4875-b5f7-c91ef8f20487"}
{"level":"info","ts":"2026-06-11T10:12:30.847Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"00181db5-aea4-4875-b5f7-c91ef8f20487","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:33.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"81a90a46-7d00-45eb-a008-f03b099cf045","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:33.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"81a90a46-7d00-45eb-a008-f03b099cf045","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:33.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"81a90a46-7d00-45eb-a008-f03b099cf045","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:33.849Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"81a90a46-7d00-45eb-a008-f03b099cf045"}
{"level":"info","ts":"2026-06-11T10:12:33.849Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"81a90a46-7d00-45eb-a008-f03b099cf045","RayCluster":"counter-rpnf8","State":""}
{"level":"info","ts":"2026-06-11T10:12:36.549Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4487f69a-639f-45c2-9797-2cb2c0cd1f06"}
{"level":"info","ts":"2026-06-11T10:12:36.549Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4487f69a-639f-45c2-9797-2cb2c0cd1f06"}
{"level":"info","ts":"2026-06-11T10:12:36.549Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4487f69a-639f-45c2-9797-2cb2c0cd1f06","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:36.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4487f69a-639f-45c2-9797-2cb2c0cd1f06","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:36.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4487f69a-639f-45c2-9797-2cb2c0cd1f06","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:36.549Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4487f69a-639f-45c2-9797-2cb2c0cd1f06","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:12:36.549Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"4487f69a-639f-45c2-9797-2cb2c0cd1f06","name":"counter-rpnf8","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:36.562Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"d9cf9042-409d-4f0e-a18b-faaa290cced2","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:36.562Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"90abde48-36eb-4fc8-b7b3-e4657719f535","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:36.562Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"d9cf9042-409d-4f0e-a18b-faaa290cced2","pod":"counter-rpnf8-head-m7x9j","ip":"10.133.0.18"}
{"level":"info","ts":"2026-06-11T10:12:36.562Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"d9cf9042-409d-4f0e-a18b-faaa290cced2","rayCluster":"counter-rpnf8","podIPs":["10.133.0.18"]}
{"level":"info","ts":"2026-06-11T10:12:36.562Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:36.562Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:36.562Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:36.563Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f"}
{"level":"info","ts":"2026-06-11T10:12:36.563Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","head service name":"counter-rpnf8-head-svc","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:36.563Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","head service URL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:12:36.563Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","SubmissionMode":"HTTPMode","RayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:36.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:12:36.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"1f6dd8e9-5de2-4f48-bdce-2e6586a1e12f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:12:36.576Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"bd66fc48-59c9-44c9-a20e-55546a3a7a97","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:36.576Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"bd66fc48-59c9-44c9-a20e-55546a3a7a97","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:36.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"bd66fc48-59c9-44c9-a20e-55546a3a7a97","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:36.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"bd66fc48-59c9-44c9-a20e-55546a3a7a97"}
{"level":"info","ts":"2026-06-11T10:12:36.585Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"90abde48-36eb-4fc8-b7b3-e4657719f535","name":"counter-rpnf8-head"}
{"level":"info","ts":"2026-06-11T10:12:36.588Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"bd66fc48-59c9-44c9-a20e-55546a3a7a97","JobId":"counter-rmmdb"}
{"level":"info","ts":"2026-06-11T10:12:36.588Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"bd66fc48-59c9-44c9-a20e-55546a3a7a97","rayJob":"counter","jobInfo":"{\"runtime_env\":{\"env_vars\":{\"counter_name\":\"test_counter\"}},\"entrypoint_resources\":{\"R1\":2},\"entrypoint\":\"python /home/ray/jobs/counter.py\",\"submission_id\":\"counter-rmmdb\",\"entrypoint_num_cpus\":2,\"entrypoint_num_gpus\":2}"}
{"level":"info","ts":"2026-06-11T10:12:36.599Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"90abde48-36eb-4fc8-b7b3-e4657719f535","name":"counter-rpnf8-workers"}
{"level":"info","ts":"2026-06-11T10:12:36.599Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"90abde48-36eb-4fc8-b7b3-e4657719f535","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:38.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"23e8e477-46ac-458a-b792-94bfbba3fbc4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:38.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"23e8e477-46ac-458a-b792-94bfbba3fbc4","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:38.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"23e8e477-46ac-458a-b792-94bfbba3fbc4","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:38.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"23e8e477-46ac-458a-b792-94bfbba3fbc4"}
{"level":"info","ts":"2026-06-11T10:12:38.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"23e8e477-46ac-458a-b792-94bfbba3fbc4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:12:38.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"23e8e477-46ac-458a-b792-94bfbba3fbc4","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:12:38.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"98300b77-2d0d-4895-baa2-911d2100a1e5","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:38.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"98300b77-2d0d-4895-baa2-911d2100a1e5","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:38.026Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"98300b77-2d0d-4895-baa2-911d2100a1e5","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:38.026Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"98300b77-2d0d-4895-baa2-911d2100a1e5"}
{"level":"info","ts":"2026-06-11T10:12:38.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"98300b77-2d0d-4895-baa2-911d2100a1e5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:12:38.563Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"b53a0c63-243f-4b46-a395-b28c755ce53e"}
{"level":"info","ts":"2026-06-11T10:12:38.563Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"b53a0c63-243f-4b46-a395-b28c755ce53e"}
{"level":"info","ts":"2026-06-11T10:12:38.563Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"b53a0c63-243f-4b46-a395-b28c755ce53e","1 head service found":"counter-rpnf8-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:38.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"b53a0c63-243f-4b46-a395-b28c755ce53e","Found 1 head Pod":"counter-rpnf8-head-m7x9j","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:38.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"b53a0c63-243f-4b46-a395-b28c755ce53e","head Pod":"counter-rpnf8-head-m7x9j","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-rpnf8-head-m7x9j. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:38.563Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"b53a0c63-243f-4b46-a395-b28c755ce53e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:38.563Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"b53a0c63-243f-4b46-a395-b28c755ce53e","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:41.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a6d86350-de72-4570-a2bc-40388d423e06","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:41.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a6d86350-de72-4570-a2bc-40388d423e06","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:41.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a6d86350-de72-4570-a2bc-40388d423e06","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:41.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a6d86350-de72-4570-a2bc-40388d423e06"}
{"level":"info","ts":"2026-06-11T10:12:41.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a6d86350-de72-4570-a2bc-40388d423e06","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:12:41.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a6d86350-de72-4570-a2bc-40388d423e06","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:12:41.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a8dbe647-a673-4e40-b6b7-bc2418408d7b","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:41.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a8dbe647-a673-4e40-b6b7-bc2418408d7b","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:41.026Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a8dbe647-a673-4e40-b6b7-bc2418408d7b","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:41.026Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a8dbe647-a673-4e40-b6b7-bc2418408d7b"}
{"level":"info","ts":"2026-06-11T10:12:41.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"a8dbe647-a673-4e40-b6b7-bc2418408d7b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:12:44.025Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"03c17532-92d5-48ad-8062-6dd7128bb842","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:44.025Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"03c17532-92d5-48ad-8062-6dd7128bb842","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.025Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"03c17532-92d5-48ad-8062-6dd7128bb842","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.025Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"03c17532-92d5-48ad-8062-6dd7128bb842"}
{"level":"info","ts":"2026-06-11T10:12:44.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"03c17532-92d5-48ad-8062-6dd7128bb842","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:12:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:12:37Z","endTime":"2026-06-11T10:12:42Z"},"jobId":"counter-rmmdb","rayClusterName":"counter-rpnf8","dashboardURL":"counter-rpnf8-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-11T10:12:15Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.18","serviceIP":"10.133.0.18","podName":"counter-rpnf8-head-m7x9j","serviceName":"counter-rpnf8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:12:44.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"03c17532-92d5-48ad-8062-6dd7128bb842","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-11T10:12:44.045Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"7beb5d1d-16ea-4a02-9cbb-705dd36995a7","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:44.045Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"7beb5d1d-16ea-4a02-9cbb-705dd36995a7","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:12:44 +0000 UTC","Now":"2026-06-11T10:12:44.045Z","ShutdownTime":"2026-06-11T10:12:44.000Z"}
{"level":"info","ts":"2026-06-11T10:12:44.045Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"7beb5d1d-16ea-4a02-9cbb-705dd36995a7","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:12:44.056Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"7beb5d1d-16ea-4a02-9cbb-705dd36995a7","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.056Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"7beb5d1d-16ea-4a02-9cbb-705dd36995a7","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:12:44.056Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"7beb5d1d-16ea-4a02-9cbb-705dd36995a7","RayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.057Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"05e375b3-c245-4704-9318-3aabab19ed17","rayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.057Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.057Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"05e375b3-c245-4704-9318-3aabab19ed17","pod":"counter-rpnf8-head-m7x9j","ip":"10.133.0.18"}
{"level":"info","ts":"2026-06-11T10:12:44.057Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"76822e43-45fa-4e72-9f3c-bf980d7dcfad"}
{"level":"info","ts":"2026-06-11T10:12:44.057Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"05e375b3-c245-4704-9318-3aabab19ed17","rayCluster":"counter-rpnf8","podIPs":["10.133.0.18"]}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"reconcileID":"9c42b317-7946-481e-9446-ff0ff8550af2"}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","namespace":"test-ns-ccphs","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"8cfc8930-30a0-4aa2-b292-5bc49bb5cc2a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"8cfc8930-30a0-4aa2-b292-5bc49bb5cc2a","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:12:44 +0000 UTC","Now":"2026-06-11T10:12:44.058Z","ShutdownTime":"2026-06-11T10:12:44.000Z"}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"8cfc8930-30a0-4aa2-b292-5bc49bb5cc2a","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"8cfc8930-30a0-4aa2-b292-5bc49bb5cc2a","RayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"8cfc8930-30a0-4aa2-b292-5bc49bb5cc2a","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:12:44.058Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"8cfc8930-30a0-4aa2-b292-5bc49bb5cc2a","RayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.064Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","name":"kuberay-gateway-access","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:44.070Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","name":"test-ns-ccphs-counter-rpnf8","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:12:44.075Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","configMap":"kube-rbac-proxy-config-counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.080Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","serviceAccount":"counter-rpnf8-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:12:44.088Z","logger":"raycluster-resource","msg":"default","name":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.088Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-rpnf8","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:44.098Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"0ced1ad5-b117-4e6b-8bc0-f84f101119de","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.098Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.098Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.098Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465","cluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.098Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465","namespace":"test-ns-ccphs","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:12:44.098Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"28695a77-193b-4b8b-ad6b-d52f750195ba","namespace":"test-ns-ccphs","clusterName":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f9024ec2-365f-4842-a1a6-e27a1d30bbe9"}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"f83cd4fa-a6ef-4f52-bfdb-f0e27e0fa230","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"f83cd4fa-a6ef-4f52-bfdb-f0e27e0fa230","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:12:44 +0000 UTC","Now":"2026-06-11T10:12:44.099Z","ShutdownTime":"2026-06-11T10:12:44.000Z"}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"f83cd4fa-a6ef-4f52-bfdb-f0e27e0fa230","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"f83cd4fa-a6ef-4f52-bfdb-f0e27e0fa230","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"f83cd4fa-a6ef-4f52-bfdb-f0e27e0fa230","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:12:44.099Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"f83cd4fa-a6ef-4f52-bfdb-f0e27e0fa230","RayCluster":"counter-rpnf8"}
{"level":"error","ts":"2026-06-11T10:12:44.105Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465","error":"rayclusters.ray.io \"counter-rpnf8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:44.105Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465"}
{"level":"error","ts":"2026-06-11T10:12:44.105Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"fabc9dae-5ff7-4765-aea3-5412757a4465","error":"rayclusters.ray.io \"counter-rpnf8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:44.105Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"82f031af-c2b2-4cf9-a015-d9228451ee7f","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.105Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"82f031af-c2b2-4cf9-a015-d9228451ee7f"}
{"level":"info","ts":"2026-06-11T10:12:44.105Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"82f031af-c2b2-4cf9-a015-d9228451ee7f","namespace":"test-ns-ccphs","authClusters":0}
{"level":"info","ts":"2026-06-11T10:12:44.105Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"82f031af-c2b2-4cf9-a015-d9228451ee7f","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:44.109Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"4b435c92-c53e-4b9d-892b-44635a3414b1"}
{"level":"info","ts":"2026-06-11T10:12:44.110Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f2439856-562c-44d6-be56-e74033988b0e","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f2439856-562c-44d6-be56-e74033988b0e"}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f2439856-562c-44d6-be56-e74033988b0e","namespace":"test-ns-ccphs","authClusters":0}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"f2439856-562c-44d6-be56-e74033988b0e","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"ff6ba91a-ded5-4a17-ace1-c7545559d455","namespacedName":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"ff6ba91a-ded5-4a17-ace1-c7545559d455"}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"ff6ba91a-ded5-4a17-ace1-c7545559d455","namespace":"test-ns-ccphs","authClusters":0}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"ff6ba91a-ded5-4a17-ace1-c7545559d455","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:44.127Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"21f93d90-41d4-426e-a187-f16317a810d8"}
{"level":"error","ts":"2026-06-11T10:12:44.127Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"28695a77-193b-4b8b-ad6b-d52f750195ba","issuer":"ray-selfsigned-issuer-counter-rpnf8","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-rpnf8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:12:44.131Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"28695a77-193b-4b8b-ad6b-d52f750195ba","certificate":"ray-head-cert-counter-rpnf8","error":"certificates.cert-manager.io \"ray-head-cert-counter-rpnf8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:44.159Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"28695a77-193b-4b8b-ad6b-d52f750195ba","errorCount":2}
{"level":"info","ts":"2026-06-11T10:12:44.159Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"28695a77-193b-4b8b-ad6b-d52f750195ba"}
{"level":"error","ts":"2026-06-11T10:12:44.159Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"28695a77-193b-4b8b-ad6b-d52f750195ba","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-rpnf8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:44.165Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"7292ac6c-e51c-4c8d-9e71-3aac9f79932b","namespace":"test-ns-ccphs","clusterName":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:44.165Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"counter-rpnf8","reconcileID":"7292ac6c-e51c-4c8d-9e71-3aac9f79932b","namespace":"test-ns-ccphs","clusterName":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:45.031Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"595ef69e-975f-43ee-aca6-040f9051563b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:45.031Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"595ef69e-975f-43ee-aca6-040f9051563b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:12:45.040Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"595ef69e-975f-43ee-aca6-040f9051563b"}
{"level":"info","ts":"2026-06-11T10:12:45.040Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"595ef69e-975f-43ee-aca6-040f9051563b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:12:45.040Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"595ef69e-975f-43ee-aca6-040f9051563b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:12:45.040Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"595ef69e-975f-43ee-aca6-040f9051563b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:12:45.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"443669a9-0938-4196-b4c1-c11b5cd1ccec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:45.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"443669a9-0938-4196-b4c1-c11b5cd1ccec","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.052Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"443669a9-0938-4196-b4c1-c11b5cd1ccec","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.052Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"443669a9-0938-4196-b4c1-c11b5cd1ccec","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.058Z","logger":"raycluster-resource","msg":"default","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.058Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-s9zgk","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:45.069Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"443669a9-0938-4196-b4c1-c11b5cd1ccec","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"443669a9-0938-4196-b4c1-c11b5cd1ccec"}
{"level":"info","ts":"2026-06-11T10:12:45.069Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"443669a9-0938-4196-b4c1-c11b5cd1ccec","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:45.069Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5621a1f-6931-4dc4-a487-f57be7eb1e58","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.069Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","mode":"IntegratedOAuth","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"44aafe98-dc40-403d-97ab-7cc84e129d2d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"44aafe98-dc40-403d-97ab-7cc84e129d2d","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"44aafe98-dc40-403d-97ab-7cc84e129d2d","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.069Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"44aafe98-dc40-403d-97ab-7cc84e129d2d"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"44aafe98-dc40-403d-97ab-7cc84e129d2d","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c012cfa4-49e2-4174-a831-d8566b599d05","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c012cfa4-49e2-4174-a831-d8566b599d05","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c012cfa4-49e2-4174-a831-d8566b599d05","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c012cfa4-49e2-4174-a831-d8566b599d05"}
{"level":"info","ts":"2026-06-11T10:12:45.070Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c012cfa4-49e2-4174-a831-d8566b599d05","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:45.077Z","logger":"raycluster-resource","msg":"default","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.077Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-s9zgk","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:12:45.128Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.128Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.128Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","name":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:45.128Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"error","ts":"2026-06-11T10:12:45.128Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.128Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.129Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.227Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:12:45.227Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","name":"fail-s9zgk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:45.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6ea11097-099b-4d8d-9c84-8a29fa4c1926","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:45.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6ea11097-099b-4d8d-9c84-8a29fa4c1926","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6ea11097-099b-4d8d-9c84-8a29fa4c1926","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6ea11097-099b-4d8d-9c84-8a29fa4c1926"}
{"level":"info","ts":"2026-06-11T10:12:45.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6ea11097-099b-4d8d-9c84-8a29fa4c1926","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5621a1f-6931-4dc4-a487-f57be7eb1e58","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","name":"fail-s9zgk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-s9zgk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccphs","name":"fail-s9zgk"}}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3d27694f-41e0-46cf-851a-d683a1ea1012","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e"}
{"level":"error","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"8dae6fdd-c4fd-4267-9937-5e65ddb8483e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","name":"fail-s9zgk-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"error","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:12:45.328Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","name":"fail-s9zgk","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","name":"kuberay-gateway-access","namespace":"test-ns-ccphs","operation":"created"}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7"}
{"level":"error","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6d897f9f-140e-4ed1-be1c-b967d7b0f6b7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140"}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140"}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"error","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.428Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a52d5ebb-3f93-4734-a543-b149a7d30024","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a52d5ebb-3f93-4734-a543-b149a7d30024","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a52d5ebb-3f93-4734-a543-b149a7d30024","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140"}
{"level":"error","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cc9cac07-d0a3-4ebb-bcc5-22b2f2e9c140","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a52d5ebb-3f93-4734-a543-b149a7d30024"}
{"level":"info","ts":"2026-06-11T10:12:45.429Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a52d5ebb-3f93-4734-a543-b149a7d30024","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:45.433Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3d27694f-41e0-46cf-851a-d683a1ea1012","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:12:45.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114"}
{"level":"info","ts":"2026-06-11T10:12:45.438Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114"}
{"level":"info","ts":"2026-06-11T10:12:45.438Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:45.438Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"error","ts":"2026-06-11T10:12:45.438Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.438Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.439Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.439Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.439Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.439Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114"}
{"level":"error","ts":"2026-06-11T10:12:45.439Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5a144337-fe73-428c-8fb2-e1e80fb36114","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.441Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","name":"test-ns-ccphs-fail-s9zgk","operation":"created"}
{"level":"info","ts":"2026-06-11T10:12:45.444Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f"}
{"level":"info","ts":"2026-06-11T10:12:45.445Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:12:45.446Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","name":"kube-rbac-proxy-config-fail-s9zgk","operation":"created"}
{"level":"info","ts":"2026-06-11T10:12:45.449Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3d27694f-41e0-46cf-851a-d683a1ea1012","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:12:45.449Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3d27694f-41e0-46cf-851a-d683a1ea1012","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.449Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4c5f9397-526f-4903-baf8-24b36194bf8e","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f"}
{"level":"error","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78"}
{"level":"error","ts":"2026-06-11T10:12:45.527Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"cb112274-122e-4d5d-acac-d4e622c38c78","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","rayCluster":"fail-s9zgk","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:12:45.527Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-s9zgk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f"}
{"level":"error","ts":"2026-06-11T10:12:45.527Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"47469908-81dc-41b1-b572-748e43afe08f","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-s9zgk\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:12:45.527Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-s9zgk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.527Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b"}
{"level":"error","ts":"2026-06-11T10:12:45.527Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"31acc043-dcfa-4f03-8bf9-e5530871550b","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-s9zgk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.528Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5b49104-11b1-45c8-91c5-12b4d78040f2","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.528Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ed340761-2c58-4f09-8555-b04a989345a8","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.528Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ed340761-2c58-4f09-8555-b04a989345a8","mode":"IntegratedOAuth","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.528Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ed340761-2c58-4f09-8555-b04a989345a8","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.532Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4c5f9397-526f-4903-baf8-24b36194bf8e","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:12:45.541Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ed340761-2c58-4f09-8555-b04a989345a8","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.541Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ca1bda17-d79f-4b72-93f8-6ee83a5b7112","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.541Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ca1bda17-d79f-4b72-93f8-6ee83a5b7112","mode":"IntegratedOAuth","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.541Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ca1bda17-d79f-4b72-93f8-6ee83a5b7112","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.541Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2afbb806-099b-48d6-b84e-c67280b7b6ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:45.541Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2afbb806-099b-48d6-b84e-c67280b7b6ff","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.542Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2afbb806-099b-48d6-b84e-c67280b7b6ff","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:45.542Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2afbb806-099b-48d6-b84e-c67280b7b6ff"}
{"level":"info","ts":"2026-06-11T10:12:45.542Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2afbb806-099b-48d6-b84e-c67280b7b6ff","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:45.543Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5b49104-11b1-45c8-91c5-12b4d78040f2"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","1 head service found":"fail-s9zgk-head-svc"}
{"level":"error","ts":"2026-06-11T10:12:45.627Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5b49104-11b1-45c8-91c5-12b4d78040f2","error":"certificates.cert-manager.io \"ray-worker-cert-fail-s9zgk\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:12:45.627Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5b49104-11b1-45c8-91c5-12b4d78040f2","error":"certificates.cert-manager.io \"ray-worker-cert-fail-s9zgk\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"error","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5b49104-11b1-45c8-91c5-12b4d78040f2"}
{"level":"error","ts":"2026-06-11T10:12:45.627Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e5b49104-11b1-45c8-91c5-12b4d78040f2","error":"certificates.cert-manager.io \"ray-worker-cert-fail-s9zgk\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c26dad0c-0c9d-4358-9814-65d5264eb864","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734"}
{"level":"info","ts":"2026-06-11T10:12:45.628Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c26dad0c-0c9d-4358-9814-65d5264eb864"}
{"level":"error","ts":"2026-06-11T10:12:45.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82a0c3e6-3ee1-438b-b5db-2990bdd8e734","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.632Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4c5f9397-526f-4903-baf8-24b36194bf8e","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:12:45.632Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4c5f9397-526f-4903-baf8-24b36194bf8e","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.632Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d46736dc-db9c-4cff-8435-614ab1d47ffb","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.637Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ca1bda17-d79f-4b72-93f8-6ee83a5b7112","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.638Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ae05754d-04e2-47e7-b702-d5673a3ff478","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.638Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ae05754d-04e2-47e7-b702-d5673a3ff478"}
{"level":"info","ts":"2026-06-11T10:12:45.647Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d46736dc-db9c-4cff-8435-614ab1d47ffb","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:12:45.658Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d46736dc-db9c-4cff-8435-614ab1d47ffb","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:12:45.658Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d46736dc-db9c-4cff-8435-614ab1d47ffb","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1"}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1"}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"error","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1"}
{"level":"error","ts":"2026-06-11T10:12:45.789Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"ecb99be9-1a19-4f22-bf11-76d1681e3ca1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242"}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242"}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","error":"head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"error","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found"}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242"}
{"level":"error","ts":"2026-06-11T10:12:46.109Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"74b96f0c-83c6-4167-bde4-cea6f7a92242","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-s9zgk not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","mode":"IntegratedOAuth","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","cluster":"fail-s9zgk","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","cluster":"fail-s9zgk","authType":"IntegratedOAuth","serviceAccount":"fail-s9zgk-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-s9zgk","ray.io/group":"headgroup","ray.io/identifier":"fail-s9zgk-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:12:46.750Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","enabled":true}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","name":"fail-s9zgk-head-bqwsq"}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c"}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:12:46.766Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","enabled":true}
{"level":"info","ts":"2026-06-11T10:12:46.782Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","name":"fail-s9zgk-small-group-worker-gqm59"}
{"level":"info","ts":"2026-06-11T10:12:46.782Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.782Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.782Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-s9zgk-head-svc"},"newHead":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"}}
{"level":"info","ts":"2026-06-11T10:12:46.782Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b8f68123-6c6e-480d-aae8-c0e3f20e7e5c","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"b3fa24a8-2709-4e43-afd5-bb565157f8f9","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"0d6ac124-9d44-4c17-80a2-f10532b2becf","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"0d6ac124-9d44-4c17-80a2-f10532b2becf"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.795Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-s9zgk-head-svc"},"newHead":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"}}
{"level":"info","ts":"2026-06-11T10:12:46.796Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:46.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"aa963549-94bc-4bb2-ae3a-617f8714b982","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:46.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"aa963549-94bc-4bb2-ae3a-617f8714b982","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"aa963549-94bc-4bb2-ae3a-617f8714b982","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:46.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"aa963549-94bc-4bb2-ae3a-617f8714b982"}
{"level":"info","ts":"2026-06-11T10:12:46.827Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"aa963549-94bc-4bb2-ae3a-617f8714b982","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:46.834Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","name":"fail-s9zgk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-s9zgk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccphs","name":"fail-s9zgk"}}
{"level":"info","ts":"2026-06-11T10:12:46.834Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af"}
{"level":"error","ts":"2026-06-11T10:12:46.834Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"99281e39-874e-4ae8-b5d9-917c7b6a61af","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-s9zgk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:46.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9"}
{"level":"info","ts":"2026-06-11T10:12:46.834Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9"}
{"level":"info","ts":"2026-06-11T10:12:46.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:12:46.835Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"39d8531d-4c9c-454c-867d-0891bd4fbbe9","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:46.837Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"b3fa24a8-2709-4e43-afd5-bb565157f8f9","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:12:46.849Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:46.850Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"f39f6503-e832-4801-86bc-2553d862c8e2","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0072aeea-a93a-4483-9050-2269178bde25","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:46.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0072aeea-a93a-4483-9050-2269178bde25","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.850Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"f39f6503-e832-4801-86bc-2553d862c8e2"}
{"level":"info","ts":"2026-06-11T10:12:46.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0072aeea-a93a-4483-9050-2269178bde25","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:46.852Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"b3fa24a8-2709-4e43-afd5-bb565157f8f9","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:12:46.852Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"b3fa24a8-2709-4e43-afd5-bb565157f8f9","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.852Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"23a99f10-7a5d-4f18-911a-aea3711007e5","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:46.927Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0072aeea-a93a-4483-9050-2269178bde25"}
{"level":"info","ts":"2026-06-11T10:12:46.927Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0072aeea-a93a-4483-9050-2269178bde25","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:46.927Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","name":"fail-s9zgk","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-s9zgk\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccphs","name":"fail-s9zgk"}}
{"level":"info","ts":"2026-06-11T10:12:46.927Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c"}
{"level":"error","ts":"2026-06-11T10:12:46.927Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3a3d137d-dbab-4b2e-8128-7658df5cbc1c","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-s9zgk\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:12:46.932Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2"}
{"level":"info","ts":"2026-06-11T10:12:46.932Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2"}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:46.933Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"45133411-cefb-4d02-953f-6dfb4f2cbda2","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:46.940Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"23a99f10-7a5d-4f18-911a-aea3711007e5","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:12:46.950Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"23a99f10-7a5d-4f18-911a-aea3711007e5","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:12:46.950Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"23a99f10-7a5d-4f18-911a-aea3711007e5","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:47.045Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"86e41257-56c8-4b46-b4f2-d1a361c1def4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:47.045Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"86e41257-56c8-4b46-b4f2-d1a361c1def4","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:12:44 +0000 UTC","Now":"2026-06-11T10:12:47.045Z","ShutdownTime":"2026-06-11T10:12:44.000Z"}
{"level":"info","ts":"2026-06-11T10:12:47.045Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"86e41257-56c8-4b46-b4f2-d1a361c1def4","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:12:47.045Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"86e41257-56c8-4b46-b4f2-d1a361c1def4","RayCluster":{"name":"counter-rpnf8","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:47.045Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"86e41257-56c8-4b46-b4f2-d1a361c1def4","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:12:47.045Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"86e41257-56c8-4b46-b4f2-d1a361c1def4","RayCluster":"counter-rpnf8"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:47.229Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6211088f-e714-4c4e-8627-a8c7e7c86716","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:47.262Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f3a7c89d-373d-42c3-8c3b-b185ccdbe672","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.615Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"newHead":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"}}
{"level":"info","ts":"2026-06-11T10:12:47.616Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0b6ef7fd-f879-4659-bab5-3260ac132279","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:12:47.633Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"63866567-f489-4f25-87bb-9f53fca3ae9d","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:47.633Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3ad5d057-5a46-43e5-a0ff-92796d4a1b23","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:47.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"94175248-6ffe-4933-8566-372b71c77701","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:47.633Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"63866567-f489-4f25-87bb-9f53fca3ae9d","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:12:47.633Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"94175248-6ffe-4933-8566-372b71c77701","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:47.633Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"94175248-6ffe-4933-8566-372b71c77701","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:47.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"94175248-6ffe-4933-8566-372b71c77701"}
{"level":"info","ts":"2026-06-11T10:12:47.634Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"94175248-6ffe-4933-8566-372b71c77701","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:47.648Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"63866567-f489-4f25-87bb-9f53fca3ae9d","certificate":"ray-head-cert-fail-s9zgk","dnsNames":["fail-s9zgk-head-svc","localhost","fail-s9zgk-head-svc.test-ns-ccphs.svc","fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:12:47.653Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3ad5d057-5a46-43e5-a0ff-92796d4a1b23","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:12:47.665Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"63866567-f489-4f25-87bb-9f53fca3ae9d","certificate":"ray-worker-cert-fail-s9zgk","dnsNames":["fail-s9zgk-worker-svc","localhost","fail-s9zgk-small-group","fail-s9zgk-worker-svc.test-ns-ccphs.svc","fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*-worker-*.test-ns-ccphs.svc","*-worker-*.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:12:47.666Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"63866567-f489-4f25-87bb-9f53fca3ae9d","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19"]}
{"level":"info","ts":"2026-06-11T10:12:47.673Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3ad5d057-5a46-43e5-a0ff-92796d4a1b23","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:12:47.673Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3ad5d057-5a46-43e5-a0ff-92796d4a1b23","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:47.764Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76"}
{"level":"info","ts":"2026-06-11T10:12:47.764Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76"}
{"level":"info","ts":"2026-06-11T10:12:47.764Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:47.765Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"50843989-2e81-48af-8073-fb6863a6ac76","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:48.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"b0f5cd50-f51b-4f99-995e-d1311dc56407","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:48.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"b0f5cd50-f51b-4f99-995e-d1311dc56407","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:48.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"b0f5cd50-f51b-4f99-995e-d1311dc56407","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:48.052Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"b0f5cd50-f51b-4f99-995e-d1311dc56407"}
{"level":"info","ts":"2026-06-11T10:12:48.052Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"b0f5cd50-f51b-4f99-995e-d1311dc56407","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325"}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325"}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:49.634Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:12:49.635Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:12:49.635Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5049ca6c-6f1a-417f-8349-3cba2c546325","seconds":300}
{"level":"info","ts":"2026-06-11T10:12:51.053Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1d1edec2-55de-49ef-a2da-5b5c1eee0197","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:51.053Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1d1edec2-55de-49ef-a2da-5b5c1eee0197","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:51.053Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1d1edec2-55de-49ef-a2da-5b5c1eee0197","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:51.053Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1d1edec2-55de-49ef-a2da-5b5c1eee0197"}
{"level":"info","ts":"2026-06-11T10:12:51.053Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1d1edec2-55de-49ef-a2da-5b5c1eee0197","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:54.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a3a6fed9-bb6a-4d9c-bce0-933c4ac920ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:54.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a3a6fed9-bb6a-4d9c-bce0-933c4ac920ff","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:54.054Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a3a6fed9-bb6a-4d9c-bce0-933c4ac920ff","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:54.054Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a3a6fed9-bb6a-4d9c-bce0-933c4ac920ff"}
{"level":"info","ts":"2026-06-11T10:12:54.054Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a3a6fed9-bb6a-4d9c-bce0-933c4ac920ff","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:12:57.055Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7753758b-05f4-4fcf-8403-f7349c75a8f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:12:57.055Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7753758b-05f4-4fcf-8403-f7349c75a8f6","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:12:57.055Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7753758b-05f4-4fcf-8403-f7349c75a8f6","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:12:57.055Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7753758b-05f4-4fcf-8403-f7349c75a8f6"}
{"level":"info","ts":"2026-06-11T10:12:57.055Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7753758b-05f4-4fcf-8403-f7349c75a8f6","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:00.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cd282032-41d9-4777-a227-26f0062729cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:00.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cd282032-41d9-4777-a227-26f0062729cb","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:00.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cd282032-41d9-4777-a227-26f0062729cb","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:00.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cd282032-41d9-4777-a227-26f0062729cb"}
{"level":"info","ts":"2026-06-11T10:13:00.056Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cd282032-41d9-4777-a227-26f0062729cb","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:03.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e549d2de-1cbc-4a8d-9bc5-6c60b8b1cc44","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:03.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e549d2de-1cbc-4a8d-9bc5-6c60b8b1cc44","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:03.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e549d2de-1cbc-4a8d-9bc5-6c60b8b1cc44","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:03.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e549d2de-1cbc-4a8d-9bc5-6c60b8b1cc44"}
{"level":"info","ts":"2026-06-11T10:13:03.056Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e549d2de-1cbc-4a8d-9bc5-6c60b8b1cc44","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:06.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cf72eae7-d7cb-4000-b452-ec808d69d747","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:06.057Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cf72eae7-d7cb-4000-b452-ec808d69d747","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:06.057Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cf72eae7-d7cb-4000-b452-ec808d69d747","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:06.057Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cf72eae7-d7cb-4000-b452-ec808d69d747"}
{"level":"info","ts":"2026-06-11T10:13:06.057Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cf72eae7-d7cb-4000-b452-ec808d69d747","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:09.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"18bafe44-95ec-4430-9719-4cff8a5066e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:09.057Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"18bafe44-95ec-4430-9719-4cff8a5066e5","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:09.057Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"18bafe44-95ec-4430-9719-4cff8a5066e5","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:09.058Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"18bafe44-95ec-4430-9719-4cff8a5066e5"}
{"level":"info","ts":"2026-06-11T10:13:09.058Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"18bafe44-95ec-4430-9719-4cff8a5066e5","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:12.058Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"839c4f70-ffec-4c00-9c5d-9bebe08efb14","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:12.058Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"839c4f70-ffec-4c00-9c5d-9bebe08efb14","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:12.058Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"839c4f70-ffec-4c00-9c5d-9bebe08efb14","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:12.059Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"839c4f70-ffec-4c00-9c5d-9bebe08efb14"}
{"level":"info","ts":"2026-06-11T10:13:12.059Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"839c4f70-ffec-4c00-9c5d-9bebe08efb14","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:15.059Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"55b8efd7-d15d-4396-b678-a31d179cdd7e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:15.059Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"55b8efd7-d15d-4396-b678-a31d179cdd7e","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:15.059Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"55b8efd7-d15d-4396-b678-a31d179cdd7e","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:15.059Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"55b8efd7-d15d-4396-b678-a31d179cdd7e"}
{"level":"info","ts":"2026-06-11T10:13:15.059Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"55b8efd7-d15d-4396-b678-a31d179cdd7e","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:15.639Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"eb6612a2-9140-4de5-bf0a-082373a8aef7","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:15.639Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"eb6612a2-9140-4de5-bf0a-082373a8aef7","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:13:15.639Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"eb6612a2-9140-4de5-bf0a-082373a8aef7","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:13:15.653Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"eb6612a2-9140-4de5-bf0a-082373a8aef7","certificate":"ray-head-cert-fail-s9zgk","dnsNames":["fail-s9zgk-head-svc","localhost","fail-s9zgk-head-svc.test-ns-ccphs.svc","fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.19","10.134.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:13:15.664Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"eb6612a2-9140-4de5-bf0a-082373a8aef7","certificate":"ray-worker-cert-fail-s9zgk","dnsNames":["fail-s9zgk-worker-svc","localhost","fail-s9zgk-small-group","fail-s9zgk-worker-svc.test-ns-ccphs.svc","fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*-worker-*.test-ns-ccphs.svc","*-worker-*.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.19","10.134.0.26","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:13:15.664Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"eb6612a2-9140-4de5-bf0a-082373a8aef7","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19","10.134.0.26"]}
{"level":"info","ts":"2026-06-11T10:13:16.581Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"1a893fa8-f6bb-49d2-aaad-b40ac9d25e2e","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:13:16.581Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"1a893fa8-f6bb-49d2-aaad-b40ac9d25e2e","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:13:16.581Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"1a893fa8-f6bb-49d2-aaad-b40ac9d25e2e","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:13:16.581Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"1a893fa8-f6bb-49d2-aaad-b40ac9d25e2e","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:13:18.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"adaf86ee-2a83-46ea-9c55-6cd8d56ecaee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:18.060Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"adaf86ee-2a83-46ea-9c55-6cd8d56ecaee","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:18.060Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"adaf86ee-2a83-46ea-9c55-6cd8d56ecaee","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:18.060Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"adaf86ee-2a83-46ea-9c55-6cd8d56ecaee"}
{"level":"info","ts":"2026-06-11T10:13:18.060Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"adaf86ee-2a83-46ea-9c55-6cd8d56ecaee","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:21.061Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"868966f0-443e-45aa-b980-c3ae50be8c02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:21.061Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"868966f0-443e-45aa-b980-c3ae50be8c02","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:21.061Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"868966f0-443e-45aa-b980-c3ae50be8c02","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:21.061Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"868966f0-443e-45aa-b980-c3ae50be8c02"}
{"level":"info","ts":"2026-06-11T10:13:21.061Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"868966f0-443e-45aa-b980-c3ae50be8c02","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:24.061Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"db60e2a6-7098-4964-8904-54ef35f8759d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:24.061Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"db60e2a6-7098-4964-8904-54ef35f8759d","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:24.061Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"db60e2a6-7098-4964-8904-54ef35f8759d","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:24.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"db60e2a6-7098-4964-8904-54ef35f8759d"}
{"level":"info","ts":"2026-06-11T10:13:24.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"db60e2a6-7098-4964-8904-54ef35f8759d","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:27.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d464d8ae-cf2d-4672-be85-ac323e74ee32","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:27.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d464d8ae-cf2d-4672-be85-ac323e74ee32","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:27.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d464d8ae-cf2d-4672-be85-ac323e74ee32","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:27.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d464d8ae-cf2d-4672-be85-ac323e74ee32"}
{"level":"info","ts":"2026-06-11T10:13:27.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d464d8ae-cf2d-4672-be85-ac323e74ee32","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:30.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"60b8064b-6e9c-4c56-873f-e0308f0581bb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:30.063Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"60b8064b-6e9c-4c56-873f-e0308f0581bb","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:30.063Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"60b8064b-6e9c-4c56-873f-e0308f0581bb","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:30.063Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"60b8064b-6e9c-4c56-873f-e0308f0581bb"}
{"level":"info","ts":"2026-06-11T10:13:30.063Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"60b8064b-6e9c-4c56-873f-e0308f0581bb","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:33.064Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e9574003-789b-45ca-83be-1fcbea9127e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:33.064Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e9574003-789b-45ca-83be-1fcbea9127e6","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:33.064Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e9574003-789b-45ca-83be-1fcbea9127e6","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:33.064Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e9574003-789b-45ca-83be-1fcbea9127e6"}
{"level":"info","ts":"2026-06-11T10:13:33.064Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e9574003-789b-45ca-83be-1fcbea9127e6","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:36.065Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c7964f90-ec89-40cd-8cfd-b7369bbc670c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:36.065Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c7964f90-ec89-40cd-8cfd-b7369bbc670c","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:36.065Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c7964f90-ec89-40cd-8cfd-b7369bbc670c","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:36.065Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c7964f90-ec89-40cd-8cfd-b7369bbc670c"}
{"level":"info","ts":"2026-06-11T10:13:36.065Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c7964f90-ec89-40cd-8cfd-b7369bbc670c","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:39.066Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0d07918-13cd-479b-8dda-db9d6e28be79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:39.066Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0d07918-13cd-479b-8dda-db9d6e28be79","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:39.066Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0d07918-13cd-479b-8dda-db9d6e28be79","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:39.066Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0d07918-13cd-479b-8dda-db9d6e28be79"}
{"level":"info","ts":"2026-06-11T10:13:39.066Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0d07918-13cd-479b-8dda-db9d6e28be79","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:42.066Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"82c5edf0-0277-4e1a-bf3e-4271cccfe593","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:42.066Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"82c5edf0-0277-4e1a-bf3e-4271cccfe593","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:42.066Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"82c5edf0-0277-4e1a-bf3e-4271cccfe593","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:42.067Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"82c5edf0-0277-4e1a-bf3e-4271cccfe593"}
{"level":"info","ts":"2026-06-11T10:13:42.067Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"82c5edf0-0277-4e1a-bf3e-4271cccfe593","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:45.067Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2ca83ae0-8213-430b-acab-6dd080571c02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:45.067Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2ca83ae0-8213-430b-acab-6dd080571c02","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:45.067Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2ca83ae0-8213-430b-acab-6dd080571c02","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:45.067Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2ca83ae0-8213-430b-acab-6dd080571c02"}
{"level":"info","ts":"2026-06-11T10:13:45.067Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"2ca83ae0-8213-430b-acab-6dd080571c02","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:48.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"99248303-203c-41da-bcf5-f5c36b47b70c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:48.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"99248303-203c-41da-bcf5-f5c36b47b70c","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:48.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"99248303-203c-41da-bcf5-f5c36b47b70c","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:48.068Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"99248303-203c-41da-bcf5-f5c36b47b70c"}
{"level":"info","ts":"2026-06-11T10:13:48.068Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"99248303-203c-41da-bcf5-f5c36b47b70c","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:51.070Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d4295c0f-f719-46cc-b4f9-48bc058b572b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:51.070Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d4295c0f-f719-46cc-b4f9-48bc058b572b","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:51.070Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d4295c0f-f719-46cc-b4f9-48bc058b572b","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:51.070Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d4295c0f-f719-46cc-b4f9-48bc058b572b"}
{"level":"info","ts":"2026-06-11T10:13:51.070Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d4295c0f-f719-46cc-b4f9-48bc058b572b","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:54.070Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0595ff26-d0f3-4eaa-8f2b-f41739a60f4e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:54.070Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0595ff26-d0f3-4eaa-8f2b-f41739a60f4e","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:54.071Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0595ff26-d0f3-4eaa-8f2b-f41739a60f4e","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:54.071Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0595ff26-d0f3-4eaa-8f2b-f41739a60f4e"}
{"level":"info","ts":"2026-06-11T10:13:54.071Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0595ff26-d0f3-4eaa-8f2b-f41739a60f4e","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:13:57.071Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0f087b01-ec64-4bab-a194-0de55a4be0e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:13:57.071Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0f087b01-ec64-4bab-a194-0de55a4be0e8","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:13:57.071Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0f087b01-ec64-4bab-a194-0de55a4be0e8","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:13:57.071Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0f087b01-ec64-4bab-a194-0de55a4be0e8"}
{"level":"info","ts":"2026-06-11T10:13:57.071Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0f087b01-ec64-4bab-a194-0de55a4be0e8","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:00.072Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4744daf8-b9bc-4db7-8666-ce82b26f70bf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:00.072Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4744daf8-b9bc-4db7-8666-ce82b26f70bf","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:00.072Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4744daf8-b9bc-4db7-8666-ce82b26f70bf","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:00.073Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4744daf8-b9bc-4db7-8666-ce82b26f70bf"}
{"level":"info","ts":"2026-06-11T10:14:00.073Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4744daf8-b9bc-4db7-8666-ce82b26f70bf","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:03.073Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"eb0d2a18-3c1a-496a-8eba-e0743b70f217","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:03.073Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"eb0d2a18-3c1a-496a-8eba-e0743b70f217","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:03.073Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"eb0d2a18-3c1a-496a-8eba-e0743b70f217","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:03.073Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"eb0d2a18-3c1a-496a-8eba-e0743b70f217"}
{"level":"info","ts":"2026-06-11T10:14:03.073Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"eb0d2a18-3c1a-496a-8eba-e0743b70f217","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:06.074Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"89d0ebc4-9760-4985-895f-ca40c18c4dcc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:06.074Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"89d0ebc4-9760-4985-895f-ca40c18c4dcc","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:06.074Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"89d0ebc4-9760-4985-895f-ca40c18c4dcc","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:06.074Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"89d0ebc4-9760-4985-895f-ca40c18c4dcc"}
{"level":"info","ts":"2026-06-11T10:14:06.074Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"89d0ebc4-9760-4985-895f-ca40c18c4dcc","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:09.075Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"23f55919-caa9-40a7-89ff-c57c8330358d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:09.075Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"23f55919-caa9-40a7-89ff-c57c8330358d","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:09.075Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"23f55919-caa9-40a7-89ff-c57c8330358d","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:09.075Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"23f55919-caa9-40a7-89ff-c57c8330358d"}
{"level":"info","ts":"2026-06-11T10:14:09.075Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"23f55919-caa9-40a7-89ff-c57c8330358d","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:12.075Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6d774386-cfd4-4c13-9701-223a5b2573de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:12.075Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6d774386-cfd4-4c13-9701-223a5b2573de","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:12.075Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6d774386-cfd4-4c13-9701-223a5b2573de","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:12.076Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6d774386-cfd4-4c13-9701-223a5b2573de"}
{"level":"info","ts":"2026-06-11T10:14:12.076Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6d774386-cfd4-4c13-9701-223a5b2573de","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:15.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1accdae3-0036-49e3-b27b-babad61ab175","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:15.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1accdae3-0036-49e3-b27b-babad61ab175","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:15.077Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1accdae3-0036-49e3-b27b-babad61ab175","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:15.077Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1accdae3-0036-49e3-b27b-babad61ab175"}
{"level":"info","ts":"2026-06-11T10:14:15.077Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"1accdae3-0036-49e3-b27b-babad61ab175","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:15.665Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"fd7877ce-85a9-46ae-a281-0cf58241c35f","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:15.665Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"fd7877ce-85a9-46ae-a281-0cf58241c35f","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:14:15.665Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"fd7877ce-85a9-46ae-a281-0cf58241c35f","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:14:15.665Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"fd7877ce-85a9-46ae-a281-0cf58241c35f","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19","10.134.0.26"]}
{"level":"info","ts":"2026-06-11T10:14:16.581Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5313c504-1abe-4561-b881-f82a4e386fa2","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:14:16.581Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5313c504-1abe-4561-b881-f82a4e386fa2","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:14:16.581Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5313c504-1abe-4561-b881-f82a4e386fa2","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:14:16.581Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"5313c504-1abe-4561-b881-f82a4e386fa2","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","podName":"fail-s9zgk-head-bqwsq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:17.949Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"1b87dcf7-f1e5-407b-87fd-db947566847b","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:18.078Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"48bf8a55-94d9-4295-9f12-94c49e8bdf94","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:18.078Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"48bf8a55-94d9-4295-9f12-94c49e8bdf94","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:18.078Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"48bf8a55-94d9-4295-9f12-94c49e8bdf94","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:18.078Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"48bf8a55-94d9-4295-9f12-94c49e8bdf94"}
{"level":"info","ts":"2026-06-11T10:14:18.078Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"48bf8a55-94d9-4295-9f12-94c49e8bdf94","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15"}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15"}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","podName":"fail-s9zgk-head-bqwsq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","podName":"fail-s9zgk-head-bqwsq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:14:18.952Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"c1d59521-1c0a-4151-a60f-aed640d4fc15","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"1ca982ed-6496-48b9-94c6-6eb3b0b67ccc","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e048416c-897f-4352-aa84-d6d420c3ffed","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"1ca982ed-6496-48b9-94c6-6eb3b0b67ccc","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"1ca982ed-6496-48b9-94c6-6eb3b0b67ccc","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4e88f43e-189b-42e1-bb2f-54ffff7a8ddc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4e88f43e-189b-42e1-bb2f-54ffff7a8ddc","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4e88f43e-189b-42e1-bb2f-54ffff7a8ddc","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"1ca982ed-6496-48b9-94c6-6eb3b0b67ccc","rayCluster":"fail-s9zgk","podIPs":["10.134.0.26","10.133.0.19"]}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4e88f43e-189b-42e1-bb2f-54ffff7a8ddc"}
{"level":"info","ts":"2026-06-11T10:14:18.965Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"4e88f43e-189b-42e1-bb2f-54ffff7a8ddc","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:18.988Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e048416c-897f-4352-aa84-d6d420c3ffed","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:14:19.000Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e048416c-897f-4352-aa84-d6d420c3ffed","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:14:19.000Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"e048416c-897f-4352-aa84-d6d420c3ffed","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2"}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2"}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","podName":"fail-s9zgk-head-bqwsq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:14:20.965Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","podName":"fail-s9zgk-head-bqwsq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:14:20.966Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:20.966Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"82d0bbb6-b102-48ca-8713-92f1c85c8ea2","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:21.078Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"3934fb9e-53c9-4f72-a4f5-4c7a504ae428","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:21.078Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"3934fb9e-53c9-4f72-a4f5-4c7a504ae428","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:21.078Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"3934fb9e-53c9-4f72-a4f5-4c7a504ae428","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:21.079Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"3934fb9e-53c9-4f72-a4f5-4c7a504ae428"}
{"level":"info","ts":"2026-06-11T10:14:21.079Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"3934fb9e-53c9-4f72-a4f5-4c7a504ae428","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:24.080Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7618f922-f900-4ca7-8972-0cf965af1ae3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:24.080Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7618f922-f900-4ca7-8972-0cf965af1ae3","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:24.080Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7618f922-f900-4ca7-8972-0cf965af1ae3","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:24.080Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7618f922-f900-4ca7-8972-0cf965af1ae3"}
{"level":"info","ts":"2026-06-11T10:14:24.080Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7618f922-f900-4ca7-8972-0cf965af1ae3","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:27.080Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbabe1c7-ef07-4beb-85ac-93f3b9a982d1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:27.080Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbabe1c7-ef07-4beb-85ac-93f3b9a982d1","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:27.080Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbabe1c7-ef07-4beb-85ac-93f3b9a982d1","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:27.080Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbabe1c7-ef07-4beb-85ac-93f3b9a982d1"}
{"level":"info","ts":"2026-06-11T10:14:27.080Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbabe1c7-ef07-4beb-85ac-93f3b9a982d1","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:14:29.986Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"abf7f3b2-c0c4-47b9-a049-0c17730beed1","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c8dd6756-9a96-4468-8a5b-bdb2c9c6b257","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"962b8569-b773-4a20-a6ea-c8e4cc3404b2","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c8dd6756-9a96-4468-8a5b-bdb2c9c6b257","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c8dd6756-9a96-4468-8a5b-bdb2c9c6b257","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"ae6f6c41-bbaf-4d5b-9551-4a32039a75e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"ae6f6c41-bbaf-4d5b-9551-4a32039a75e2","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"ae6f6c41-bbaf-4d5b-9551-4a32039a75e2","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:30.001Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c8dd6756-9a96-4468-8a5b-bdb2c9c6b257","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19","10.134.0.26"]}
{"level":"info","ts":"2026-06-11T10:14:30.002Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"ae6f6c41-bbaf-4d5b-9551-4a32039a75e2"}
{"level":"info","ts":"2026-06-11T10:14:30.002Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"ae6f6c41-bbaf-4d5b-9551-4a32039a75e2","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:30.013Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"962b8569-b773-4a20-a6ea-c8e4cc3404b2","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:14:30.030Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"962b8569-b773-4a20-a6ea-c8e4cc3404b2","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:14:30.030Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"962b8569-b773-4a20-a6ea-c8e4cc3404b2","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:30.081Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"8214ff48-e2f7-4663-b913-a17994348f13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:30.081Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"8214ff48-e2f7-4663-b913-a17994348f13","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:30.081Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"8214ff48-e2f7-4663-b913-a17994348f13","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:30.081Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"8214ff48-e2f7-4663-b913-a17994348f13"}
{"level":"info","ts":"2026-06-11T10:14:30.081Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"8214ff48-e2f7-4663-b913-a17994348f13","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:32.000Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"f7835e88-7c66-4053-af7f-a1d82d07c552","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:33.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d05f2904-9beb-4cee-a1ac-e94fb8deb811","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:33.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d05f2904-9beb-4cee-a1ac-e94fb8deb811","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:33.082Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d05f2904-9beb-4cee-a1ac-e94fb8deb811","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:33.083Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d05f2904-9beb-4cee-a1ac-e94fb8deb811"}
{"level":"info","ts":"2026-06-11T10:14:33.083Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"d05f2904-9beb-4cee-a1ac-e94fb8deb811","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:35.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba"}
{"level":"info","ts":"2026-06-11T10:14:35.217Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba"}
{"level":"info","ts":"2026-06-11T10:14:35.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:35.218Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3adaaf90-616f-4997-a9a4-283381c7f2ba","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:36.084Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbd5641d-6de1-4130-800a-bb0fed7f04f0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:36.084Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbd5641d-6de1-4130-800a-bb0fed7f04f0","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:36.084Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbd5641d-6de1-4130-800a-bb0fed7f04f0","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:36.084Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbd5641d-6de1-4130-800a-bb0fed7f04f0"}
{"level":"info","ts":"2026-06-11T10:14:36.084Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"fbd5641d-6de1-4130-800a-bb0fed7f04f0","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f"}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f"}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:14:36.214Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a8538f77-7c93-43e2-8080-3201c6620c7f","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"36dab1ff-38f0-4d30-bef9-53a05b48daa0","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"f44b783a-3c33-4edf-aaad-1a6999cd5f41","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a64534b8-1172-4d97-803e-922b5170b6af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a64534b8-1172-4d97-803e-922b5170b6af","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"f44b783a-3c33-4edf-aaad-1a6999cd5f41","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"f44b783a-3c33-4edf-aaad-1a6999cd5f41","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a64534b8-1172-4d97-803e-922b5170b6af","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"f44b783a-3c33-4edf-aaad-1a6999cd5f41","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19","10.134.0.26"]}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a64534b8-1172-4d97-803e-922b5170b6af"}
{"level":"info","ts":"2026-06-11T10:14:36.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"a64534b8-1172-4d97-803e-922b5170b6af","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:36.248Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"36dab1ff-38f0-4d30-bef9-53a05b48daa0","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:14:36.258Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"36dab1ff-38f0-4d30-bef9-53a05b48daa0","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:14:36.258Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"36dab1ff-38f0-4d30-bef9-53a05b48daa0","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:38.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17"}
{"level":"info","ts":"2026-06-11T10:14:38.228Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17"}
{"level":"info","ts":"2026-06-11T10:14:38.228Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:38.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:38.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:38.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:12:46Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:38.229Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"b82134b2-075c-45ac-816a-8428784e6b17","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:39.085Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f8e4e790-a6fd-48f2-84d2-38ca148890ef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:39.085Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f8e4e790-a6fd-48f2-84d2-38ca148890ef","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:39.085Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f8e4e790-a6fd-48f2-84d2-38ca148890ef","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:39.085Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f8e4e790-a6fd-48f2-84d2-38ca148890ef"}
{"level":"info","ts":"2026-06-11T10:14:39.085Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f8e4e790-a6fd-48f2-84d2-38ca148890ef","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:42.086Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e5a65a7d-8b22-4ddb-a8ec-b2a80c47d10e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:42.086Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e5a65a7d-8b22-4ddb-a8ec-b2a80c47d10e","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:42.086Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e5a65a7d-8b22-4ddb-a8ec-b2a80c47d10e","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:42.086Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e5a65a7d-8b22-4ddb-a8ec-b2a80c47d10e"}
{"level":"info","ts":"2026-06-11T10:14:42.086Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"e5a65a7d-8b22-4ddb-a8ec-b2a80c47d10e","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:45.087Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0c2191e-d1e2-4af6-861e-46f799a69ba8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:45.087Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0c2191e-d1e2-4af6-861e-46f799a69ba8","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:45.087Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0c2191e-d1e2-4af6-861e-46f799a69ba8","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:45.088Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0c2191e-d1e2-4af6-861e-46f799a69ba8"}
{"level":"info","ts":"2026-06-11T10:14:45.088Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"c0c2191e-d1e2-4af6-861e-46f799a69ba8","RayCluster":"fail-s9zgk","State":""}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4"}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4"}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:14:47.249Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"860a0a1c-f1f0-44b0-926a-edc28a683bd4","name":"fail-s9zgk","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:47.267Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3bc4a1e0-7441-41e5-a4b3-e67f73a33203","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:47.267Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"72913dba-1455-459a-a8ad-cafee448ffbd","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:47.267Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"72913dba-1455-459a-a8ad-cafee448ffbd","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"72913dba-1455-459a-a8ad-cafee448ffbd","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"72913dba-1455-459a-a8ad-cafee448ffbd","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19","10.134.0.26"]}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","head service name":"fail-s9zgk-head-svc","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","head service URL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","SubmissionMode":"HTTPMode","RayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","dashboardURL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:14:47.268Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"f3975c65-e232-4a38-9d13-d51e4beaae4b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:14:47.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"847519bd-5bde-4c55-a91d-47c65af51615","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:47.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"847519bd-5bde-4c55-a91d-47c65af51615","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:47.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"847519bd-5bde-4c55-a91d-47c65af51615","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:47.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"847519bd-5bde-4c55-a91d-47c65af51615"}
{"level":"info","ts":"2026-06-11T10:14:47.284Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3bc4a1e0-7441-41e5-a4b3-e67f73a33203","name":"fail-s9zgk-head"}
{"level":"info","ts":"2026-06-11T10:14:47.293Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"847519bd-5bde-4c55-a91d-47c65af51615","JobId":"fail-z7w4s"}
{"level":"info","ts":"2026-06-11T10:14:47.293Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"847519bd-5bde-4c55-a91d-47c65af51615","rayJob":"fail","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/fail.py\",\"submission_id\":\"fail-z7w4s\"}"}
{"level":"info","ts":"2026-06-11T10:14:47.303Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3bc4a1e0-7441-41e5-a4b3-e67f73a33203","name":"fail-s9zgk-workers"}
{"level":"info","ts":"2026-06-11T10:14:47.303Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3bc4a1e0-7441-41e5-a4b3-e67f73a33203","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:48.209Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"03f67c37-0303-4c5d-b1c9-29b7182a4a78","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:48.209Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"03f67c37-0303-4c5d-b1c9-29b7182a4a78","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:48.209Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"03f67c37-0303-4c5d-b1c9-29b7182a4a78","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:48.209Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"03f67c37-0303-4c5d-b1c9-29b7182a4a78"}
{"level":"info","ts":"2026-06-11T10:14:48.212Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"03f67c37-0303-4c5d-b1c9-29b7182a4a78","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","dashboardURL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:14:47Z"},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","dashboardURL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:14:48.212Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"03f67c37-0303-4c5d-b1c9-29b7182a4a78","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:14:48.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6de19508-f6e5-4c73-9d6f-5402120ff09a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:48.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6de19508-f6e5-4c73-9d6f-5402120ff09a","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:48.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6de19508-f6e5-4c73-9d6f-5402120ff09a","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:48.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6de19508-f6e5-4c73-9d6f-5402120ff09a"}
{"level":"info","ts":"2026-06-11T10:14:48.231Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"6de19508-f6e5-4c73-9d6f-5402120ff09a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:14:47Z"},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","dashboardURL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:14:47Z"},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","dashboardURL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4"}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4"}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:49.268Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38cca654-a8cf-4045-b25e-3bbb95a2f3f4","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:51.210Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"532e5479-5207-4132-ad9f-0ed589ae33b4","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.210Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"532e5479-5207-4132-ad9f-0ed589ae33b4","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:14:51.210Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"532e5479-5207-4132-ad9f-0ed589ae33b4","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.210Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"532e5479-5207-4132-ad9f-0ed589ae33b4"}
{"level":"info","ts":"2026-06-11T10:14:51.213Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"532e5479-5207-4132-ad9f-0ed589ae33b4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:14:47Z"},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","dashboardURL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:14:47Z","endTime":"2026-06-11T10:14:49Z"},"jobId":"fail-z7w4s","rayClusterName":"fail-s9zgk","dashboardURL":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:14:47,874\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-z7w4s: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:12:45Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:14:51.213Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"532e5479-5207-4132-ad9f-0ed589ae33b4","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-11T10:14:51.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"177284e6-0441-46fe-87ec-947168efaec6","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.233Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"177284e6-0441-46fe-87ec-947168efaec6","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:14:51.233Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"info","ts":"2026-06-11T10:14:51.801Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e9403902-fa0b-4faf-b798-228404d4272f","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.801Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e9403902-fa0b-4faf-b798-228404d4272f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:14:51.812Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e9403902-fa0b-4faf-b798-228404d4272f"}
{"level":"info","ts":"2026-06-11T10:14:51.812Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e9403902-fa0b-4faf-b798-228404d4272f","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:14:51.812Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e9403902-fa0b-4faf-b798-228404d4272f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:14:51.812Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e9403902-fa0b-4faf-b798-228404d4272f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:14:51.824Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.824Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea"}
{"level":"info","ts":"2026-06-11T10:14:51.824Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:14:51.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-mqsc5","rayClusterName":"stop-6pzd2","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:14:51.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:14:51.831Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:14:51.831Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea"}
{"level":"error","ts":"2026-06-11T10:14:51.831Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8efe4b90-f91c-4b6d-9ea4-c1d482474fea","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:51.831Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6304eddd-0f56-49e6-9a47-352d1902b6a7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.831Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6304eddd-0f56-49e6-9a47-352d1902b6a7","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.831Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6304eddd-0f56-49e6-9a47-352d1902b6a7","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.831Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6304eddd-0f56-49e6-9a47-352d1902b6a7","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.851Z","logger":"raycluster-resource","msg":"default","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.851Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-94xjr","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"4dd06b3c-2e9b-4dda-a150-995a157ba30a","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","namespacedName":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","mode":"IntegratedOAuth","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6304eddd-0f56-49e6-9a47-352d1902b6a7","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6304eddd-0f56-49e6-9a47-352d1902b6a7"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6304eddd-0f56-49e6-9a47-352d1902b6a7","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"afed71cf-f201-4918-91c5-9280878a7815","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"afed71cf-f201-4918-91c5-9280878a7815","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.862Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"afed71cf-f201-4918-91c5-9280878a7815","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.863Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"afed71cf-f201-4918-91c5-9280878a7815"}
{"level":"info","ts":"2026-06-11T10:14:51.863Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"afed71cf-f201-4918-91c5-9280878a7815","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:51.868Z","logger":"raycluster-resource","msg":"default","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.868Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-94xjr","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:14:51.870Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","name":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:51.870Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:51.870Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:51.871Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:51.871Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.871Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.871Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.871Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:14:51.871Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","name":"stop-94xjr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:51.927Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"4dd06b3c-2e9b-4dda-a150-995a157ba30a","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.927Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"793c3ce2-cdfd-4b38-9fee-8bcd5418a4ed","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.927Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.927Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"2e3e6b85-040f-4e7a-9a2e-ba791612eca3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"2e3e6b85-040f-4e7a-9a2e-ba791612eca3","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"2e3e6b85-040f-4e7a-9a2e-ba791612eca3","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"2e3e6b85-040f-4e7a-9a2e-ba791612eca3"}
{"level":"info","ts":"2026-06-11T10:14:51.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"2e3e6b85-040f-4e7a-9a2e-ba791612eca3","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:51.932Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","name":"stop-94xjr-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","name":"stop-94xjr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccphs","name":"stop-94xjr"}}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af"}
{"level":"error","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"65c7222a-46ed-42b4-b0c8-1139c3cb97af","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e"}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e"}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:14:51.933Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","name":"stop-94xjr","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:51.937Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"793c3ce2-cdfd-4b38-9fee-8bcd5418a4ed","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:14:51.938Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15"}
{"level":"info","ts":"2026-06-11T10:14:51.938Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:14:51.938Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.940Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","name":"test-ns-ccphs-stop-94xjr","operation":"created"}
{"level":"info","ts":"2026-06-11T10:14:51.948Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","name":"kube-rbac-proxy-config-stop-94xjr","operation":"created"}
{"level":"info","ts":"2026-06-11T10:14:51.950Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"c8e5257d-83b6-4c48-9405-8260b6a43fa0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:51.950Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"c8e5257d-83b6-4c48-9405-8260b6a43fa0","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.950Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"c8e5257d-83b6-4c48-9405-8260b6a43fa0","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"c8e5257d-83b6-4c48-9405-8260b6a43fa0"}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"c8e5257d-83b6-4c48-9405-8260b6a43fa0","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e"}
{"level":"error","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"a8be79e0-07f6-4652-bd7f-9ec710dced6e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d"}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d"}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.951Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.952Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:51.952Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d"}
{"level":"error","ts":"2026-06-11T10:14:51.952Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"5c0c8800-86d2-41e8-9907-101b65d98b9d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:51.952Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.958Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"793c3ce2-cdfd-4b38-9fee-8bcd5418a4ed","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:14:51.959Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"793c3ce2-cdfd-4b38-9fee-8bcd5418a4ed","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:51.959Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"20df9eb4-adbd-439b-bf99-d831856d67d8","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:52.027Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-11T10:14:52.027Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e4d4429d-d219-40ea-bc74-a3090d0e06b8","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264"}
{"level":"error","ts":"2026-06-11T10:14:52.027Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"70b92b04-247a-4d17-873b-820f08102264","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e2bb2af8-9563-4af4-b6a9-b773160c91f2","namespacedName":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e2bb2af8-9563-4af4-b6a9-b773160c91f2","mode":"IntegratedOAuth","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.027Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e2bb2af8-9563-4af4-b6a9-b773160c91f2","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.032Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"20df9eb4-adbd-439b-bf99-d831856d67d8","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:14:52.037Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15"}
{"level":"info","ts":"2026-06-11T10:14:52.037Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15","rayCluster":"stop-94xjr","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:14:52.037Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"36c5eaf1-e591-4250-9164-cc55d900bf15"}
{"level":"info","ts":"2026-06-11T10:14:52.037Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"29dc4172-6896-4392-a56e-eda2fbf268cf","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.037Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"29dc4172-6896-4392-a56e-eda2fbf268cf"}
{"level":"info","ts":"2026-06-11T10:14:52.040Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e2bb2af8-9563-4af4-b6a9-b773160c91f2","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"3c870265-a966-4ed9-9fd4-0a8ef9733f59","namespacedName":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"3c870265-a966-4ed9-9fd4-0a8ef9733f59","mode":"IntegratedOAuth","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"3c870265-a966-4ed9-9fd4-0a8ef9733f59","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"17e6c928-5a88-4184-a6e9-f1dc5cb9613c","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"17e6c928-5a88-4184-a6e9-f1dc5cb9613c"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"083e2386-ec85-4d41-a563-250416b1af80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"083e2386-ec85-4d41-a563-250416b1af80","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"083e2386-ec85-4d41-a563-250416b1af80","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"083e2386-ec85-4d41-a563-250416b1af80"}
{"level":"info","ts":"2026-06-11T10:14:52.041Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"083e2386-ec85-4d41-a563-250416b1af80","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:52.045Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"20df9eb4-adbd-439b-bf99-d831856d67d8","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:14:52.045Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"20df9eb4-adbd-439b-bf99-d831856d67d8","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.046Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"b5175d8a-b992-4630-b18b-585fc77dfa7a","cluster":"stop-94xjr"}
{"level":"error","ts":"2026-06-11T10:14:52.060Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"3c870265-a966-4ed9-9fd4-0a8ef9733f59","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.060Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"3c870265-a966-4ed9-9fd4-0a8ef9733f59"}
{"level":"error","ts":"2026-06-11T10:14:52.060Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"3c870265-a966-4ed9-9fd4-0a8ef9733f59","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"a8b0826e-b92b-437e-9696-868a46f8d24b","namespacedName":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"a8b0826e-b92b-437e-9696-868a46f8d24b","mode":"IntegratedOAuth","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"a8b0826e-b92b-437e-9696-868a46f8d24b","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"b5175d8a-b992-4630-b18b-585fc77dfa7a","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700"}
{"level":"error","ts":"2026-06-11T10:14:52.127Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"226d2503-8f6b-43b6-ac93-8194ec65d700","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.136Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"a8b0826e-b92b-437e-9696-868a46f8d24b","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.139Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"b5175d8a-b992-4630-b18b-585fc77dfa7a","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:14:52.140Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"b5175d8a-b992-4630-b18b-585fc77dfa7a","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7"}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7"}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7"}
{"level":"error","ts":"2026-06-11T10:14:52.208Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"d4932577-7aab-4c88-a081-65b91e9e5ed7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b"}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b"}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b"}
{"level":"error","ts":"2026-06-11T10:14:52.369Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b771b918-4e23-4e73-ac92-3afde59fea2b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521"}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521"}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521"}
{"level":"error","ts":"2026-06-11T10:14:52.690Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"39954b05-b166-4a90-93bd-a6d6e21ef521","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:53.331Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2"}
{"level":"info","ts":"2026-06-11T10:14:53.331Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2"}
{"level":"info","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","error":"head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"error","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found"}
{"level":"info","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","filter labels":["test-ns-ccphs",{"ray.io/cluster":"stop-94xjr","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2"}
{"level":"error","ts":"2026-06-11T10:14:53.332Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e62dd104-e28d-44e2-ba92-2c03b11f6df2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-94xjr not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:54.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"227469f0-9a7d-47b2-a74f-6424d9e06ac2","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:54.234Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"227469f0-9a7d-47b2-a74f-6424d9e06ac2","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:14:54.234Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","mode":"IntegratedOAuth","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","cluster":"stop-94xjr","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","cluster":"stop-94xjr","authType":"IntegratedOAuth","serviceAccount":"stop-94xjr-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-94xjr","ray.io/group":"headgroup","ray.io/identifier":"stop-94xjr-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:14:54.612Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:14:54.613Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:14:54.613Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:14:54.613Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","enabled":true}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","name":"stop-94xjr-head-r7225"}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc"}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","nodeType":"worker","rayStartParams":{"address":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","rayStartCmd":"ray start  --address=stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:14:54.632Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","enabled":true}
{"level":"info","ts":"2026-06-11T10:14:54.646Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","name":"stop-94xjr-small-group-worker-vk89n"}
{"level":"info","ts":"2026-06-11T10:14:54.646Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.646Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.646Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-94xjr-head-svc"},"newHead":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"}}
{"level":"info","ts":"2026-06-11T10:14:54.646Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"20b2369f-597a-44ab-b247-c1e0d64e21cc","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:54.659Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20"}
{"level":"info","ts":"2026-06-11T10:14:54.659Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20"}
{"level":"info","ts":"2026-06-11T10:14:54.659Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9e752c2e-c6bb-4bc1-ab00-9d0cb93380b1","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9e752c2e-c6bb-4bc1-ab00-9d0cb93380b1","rayCluster":"stop-94xjr","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"44934e8b-959e-4bce-bf4d-57798aa7a6c4","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0d951d4a-68cf-45bb-9ca5-17c6f7ae31c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0d951d4a-68cf-45bb-9ca5-17c6f7ae31c6","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0d951d4a-68cf-45bb-9ca5-17c6f7ae31c6","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-94xjr-head-svc"},"newHead":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"}}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0d951d4a-68cf-45bb-9ca5-17c6f7ae31c6"}
{"level":"info","ts":"2026-06-11T10:14:54.660Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0d951d4a-68cf-45bb-9ca5-17c6f7ae31c6","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","name":"stop-94xjr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccphs","name":"stop-94xjr"}}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20"}
{"level":"error","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9bcbb4ed-21cd-48a6-8862-89de771b8c20","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0"}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0"}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:54.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.670Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.670Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.670Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:54.670Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4ca0bb88-5ad5-4023-982f-8f2ae56e86a0","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:54.675Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"4a083dbb-1ca1-437c-ad0e-4bc1e65cdc4c","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:54.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac"}
{"level":"info","ts":"2026-06-11T10:14:54.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac"}
{"level":"info","ts":"2026-06-11T10:14:54.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:54.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:54.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:14:54.728Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"8511348d-fadb-4154-aa6a-922a8b5e5eac","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"8c0c063b-2763-4f59-8bad-96a917977e49","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"8c0c063b-2763-4f59-8bad-96a917977e49","rayCluster":"stop-94xjr","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"17310d0d-272f-4ded-918e-f73274fb1a5a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"17310d0d-272f-4ded-918e-f73274fb1a5a","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.740Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.739Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"17310d0d-272f-4ded-918e-f73274fb1a5a","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:54.740Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:14:54.740Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:54.740Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"17310d0d-272f-4ded-918e-f73274fb1a5a"}
{"level":"info","ts":"2026-06-11T10:14:54.740Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"17310d0d-272f-4ded-918e-f73274fb1a5a","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:54.742Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"44934e8b-959e-4bce-bf4d-57798aa7a6c4","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:14:54.746Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","name":"stop-94xjr","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccphs","name":"stop-94xjr"}}
{"level":"info","ts":"2026-06-11T10:14:54.746Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76"}
{"level":"error","ts":"2026-06-11T10:14:54.746Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"633d1af7-62cc-4ed7-95d5-a700da0fcc76","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-94xjr\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:14:54.752Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77"}
{"level":"info","ts":"2026-06-11T10:14:54.752Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77"}
{"level":"info","ts":"2026-06-11T10:14:54.752Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:54.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:54.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"44934e8b-959e-4bce-bf4d-57798aa7a6c4","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"44934e8b-959e-4bce-bf4d-57798aa7a6c4","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.827Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"397d7717-d8e8-4d1d-8e7e-f3804e6ebafc","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.828Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:54.828Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:54.828Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"10d6ce81-f319-4e69-a8a7-c218bb372a77","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:54.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8392e1ce-c919-4e75-9bfd-b2bf4aef8554","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:54.927Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8392e1ce-c919-4e75-9bfd-b2bf4aef8554","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:54.927Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8392e1ce-c919-4e75-9bfd-b2bf4aef8554","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:54.927Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8392e1ce-c919-4e75-9bfd-b2bf4aef8554"}
{"level":"info","ts":"2026-06-11T10:14:54.927Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8392e1ce-c919-4e75-9bfd-b2bf4aef8554","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:54.933Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"397d7717-d8e8-4d1d-8e7e-f3804e6ebafc","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:14:54.946Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"397d7717-d8e8-4d1d-8e7e-f3804e6ebafc","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:14:54.946Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"397d7717-d8e8-4d1d-8e7e-f3804e6ebafc","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:55.082Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7"}
{"level":"info","ts":"2026-06-11T10:14:55.082Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7"}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:55.083Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"1d23dd47-73fe-4bcd-9dee-b250275cf1d7","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07"}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07"}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:55.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:55.107Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:55.107Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:55.107Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:55.107Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"55165fb1-246f-49fb-b5c0-d999a3c8ef07","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"newHead":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"}}
{"level":"info","ts":"2026-06-11T10:14:55.305Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"72ffcaee-05ea-4bdc-8928-a0b82b9759fe","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:14:55.318Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"af93d765-9702-48be-8a05-0c088e384798","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:55.319Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"5cc05153-2c42-49be-b250-4ae863206677","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:55.319Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"5cc05153-2c42-49be-b250-4ae863206677","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:14:55.319Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"efe56397-efee-4e49-a0f7-4327f8d37957","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:55.319Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"efe56397-efee-4e49-a0f7-4327f8d37957","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:55.319Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"efe56397-efee-4e49-a0f7-4327f8d37957","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:55.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"efe56397-efee-4e49-a0f7-4327f8d37957"}
{"level":"info","ts":"2026-06-11T10:14:55.319Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"efe56397-efee-4e49-a0f7-4327f8d37957","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:14:55.331Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"af93d765-9702-48be-8a05-0c088e384798","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:14:55.331Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"5cc05153-2c42-49be-b250-4ae863206677","certificate":"ray-head-cert-stop-94xjr","dnsNames":["stop-94xjr-head-svc","localhost","stop-94xjr-head-svc.test-ns-ccphs.svc","stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.134.0.27","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:14:55.343Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"af93d765-9702-48be-8a05-0c088e384798","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:14:55.343Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"af93d765-9702-48be-8a05-0c088e384798","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:55.344Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"5cc05153-2c42-49be-b250-4ae863206677","certificate":"ray-worker-cert-stop-94xjr","dnsNames":["stop-94xjr-worker-svc","localhost","stop-94xjr-small-group","stop-94xjr-worker-svc.test-ns-ccphs.svc","stop-94xjr-worker-svc.test-ns-ccphs.svc.cluster.local","*.stop-94xjr-worker-svc.test-ns-ccphs.svc","*.stop-94xjr-worker-svc.test-ns-ccphs.svc.cluster.local","*-worker-*.test-ns-ccphs.svc","*-worker-*.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.134.0.27","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:14:55.344Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"5cc05153-2c42-49be-b250-4ae863206677","rayCluster":"stop-94xjr","podIPs":["10.134.0.27"]}
{"level":"info","ts":"2026-06-11T10:14:56.093Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e"}
{"level":"info","ts":"2026-06-11T10:14:56.093Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e"}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:56.094Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"e2cd3c77-2fef-46ba-b740-cc10f660de2e","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:14:57.319Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"195a995f-1f28-4337-afd7-f25f20f70467","seconds":300}
{"level":"info","ts":"2026-06-11T10:14:57.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0a007747-d394-4399-bf6e-37662f41f994","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:14:57.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0a007747-d394-4399-bf6e-37662f41f994","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:14:57.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0a007747-d394-4399-bf6e-37662f41f994","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:14:57.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0a007747-d394-4399-bf6e-37662f41f994"}
{"level":"info","ts":"2026-06-11T10:14:57.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0a007747-d394-4399-bf6e-37662f41f994","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:00.321Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09"}
{"level":"info","ts":"2026-06-11T10:15:00.321Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09"}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:00.322Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"894343d9-3a4d-4668-996c-7849e7fdaf09","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:00.929Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0c2991e9-c58a-4198-bfad-11bf72323e36","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:00.929Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0c2991e9-c58a-4198-bfad-11bf72323e36","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:00.929Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0c2991e9-c58a-4198-bfad-11bf72323e36","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:00.929Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0c2991e9-c58a-4198-bfad-11bf72323e36"}
{"level":"info","ts":"2026-06-11T10:15:00.929Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"0c2991e9-c58a-4198-bfad-11bf72323e36","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:03.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6987e6be-b88f-4048-ac90-7d37face110c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:03.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6987e6be-b88f-4048-ac90-7d37face110c","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:03.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6987e6be-b88f-4048-ac90-7d37face110c","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:03.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6987e6be-b88f-4048-ac90-7d37face110c"}
{"level":"info","ts":"2026-06-11T10:15:03.930Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6987e6be-b88f-4048-ac90-7d37face110c","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:05.140Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f648efe2-e8e2-4939-87f1-6851334b1a28","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","podName":"stop-94xjr-head-r7225","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:15:06.152Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"0567b777-1d26-44ba-9545-3c6f5138a4d5","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:06.165Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"54ac65ca-7371-46cf-a8f5-f94084701ca8","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:06.165Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"27027f10-6530-412f-b44e-ee7a5755669c","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:06.166Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"54ac65ca-7371-46cf-a8f5-f94084701ca8","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:06.166Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"54ac65ca-7371-46cf-a8f5-f94084701ca8","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:06.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"29f61516-71f3-472b-8165-f54eb361e815","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:06.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"29f61516-71f3-472b-8165-f54eb361e815","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:06.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"29f61516-71f3-472b-8165-f54eb361e815","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:06.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"29f61516-71f3-472b-8165-f54eb361e815"}
{"level":"info","ts":"2026-06-11T10:15:06.166Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"29f61516-71f3-472b-8165-f54eb361e815","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:06.178Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"54ac65ca-7371-46cf-a8f5-f94084701ca8","certificate":"ray-head-cert-stop-94xjr","dnsNames":["stop-94xjr-head-svc","localhost","stop-94xjr-head-svc.test-ns-ccphs.svc","stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.134.0.27","10.133.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:06.189Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"27027f10-6530-412f-b44e-ee7a5755669c","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:15:06.212Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"54ac65ca-7371-46cf-a8f5-f94084701ca8","certificate":"ray-worker-cert-stop-94xjr","dnsNames":["stop-94xjr-worker-svc","localhost","stop-94xjr-small-group","stop-94xjr-worker-svc.test-ns-ccphs.svc","stop-94xjr-worker-svc.test-ns-ccphs.svc.cluster.local","*.stop-94xjr-worker-svc.test-ns-ccphs.svc","*.stop-94xjr-worker-svc.test-ns-ccphs.svc.cluster.local","*-worker-*.test-ns-ccphs.svc","*-worker-*.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.134.0.27","10.133.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:06.212Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"54ac65ca-7371-46cf-a8f5-f94084701ca8","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:06.217Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"27027f10-6530-412f-b44e-ee7a5755669c","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:15:06.217Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"27027f10-6530-412f-b44e-ee7a5755669c","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:06.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8988eee9-8e28-43a2-925b-62d1dd3b9c7a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:06.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8988eee9-8e28-43a2-925b-62d1dd3b9c7a","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:06.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8988eee9-8e28-43a2-925b-62d1dd3b9c7a","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:06.945Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8988eee9-8e28-43a2-925b-62d1dd3b9c7a"}
{"level":"info","ts":"2026-06-11T10:15:06.945Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8988eee9-8e28-43a2-925b-62d1dd3b9c7a","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0"}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0"}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","podName":"stop-94xjr-head-r7225","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","podName":"stop-94xjr-head-r7225","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:15:07.389Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7f96c1ad-6e55-4cd5-8880-6ac6f61138b0","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed191ff8-7246-4649-bcd5-0960f637eff3","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"340f82cf-f8f2-4a93-8429-114afc9c3b5b","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d3d64ad6-415c-4610-8529-23aa899e1ec6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed191ff8-7246-4649-bcd5-0960f637eff3","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d3d64ad6-415c-4610-8529-23aa899e1ec6","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed191ff8-7246-4649-bcd5-0960f637eff3","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d3d64ad6-415c-4610-8529-23aa899e1ec6","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed191ff8-7246-4649-bcd5-0960f637eff3","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d3d64ad6-415c-4610-8529-23aa899e1ec6"}
{"level":"info","ts":"2026-06-11T10:15:07.402Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d3d64ad6-415c-4610-8529-23aa899e1ec6","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:07.415Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"340f82cf-f8f2-4a93-8429-114afc9c3b5b","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:15:07.425Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"340f82cf-f8f2-4a93-8429-114afc9c3b5b","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:15:07.425Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"340f82cf-f8f2-4a93-8429-114afc9c3b5b","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3"}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3"}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:08.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:08.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","podName":"stop-94xjr-head-r7225","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:08.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","podName":"stop-94xjr-head-r7225","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:08.166Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:08.166Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"84ff2c4c-8bb7-40a1-83b3-2f9de8557eb3","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:09.946Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"157c93a6-4c96-4ec3-808c-7f808518229a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:09.946Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"157c93a6-4c96-4ec3-808c-7f808518229a","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:09.946Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"157c93a6-4c96-4ec3-808c-7f808518229a","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:09.946Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"157c93a6-4c96-4ec3-808c-7f808518229a"}
{"level":"info","ts":"2026-06-11T10:15:09.946Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"157c93a6-4c96-4ec3-808c-7f808518229a","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:12.947Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"27035884-09a2-4ab9-8258-091e935b2811","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:12.947Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"27035884-09a2-4ab9-8258-091e935b2811","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:12.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"27035884-09a2-4ab9-8258-091e935b2811","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:12.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"27035884-09a2-4ab9-8258-091e935b2811"}
{"level":"info","ts":"2026-06-11T10:15:12.948Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"27035884-09a2-4ab9-8258-091e935b2811","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08"}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08"}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:15:13.418Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:15:13.419Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:15:13.419Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"82a56b90-f917-4bd5-b039-ebcc40ba5a08","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9cf4c97e-a7c7-4cea-8e4d-ddf622960c5d","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9c635aec-98ca-45e0-86c1-df107f752f03","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9cf4c97e-a7c7-4cea-8e4d-ddf622960c5d","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9cf4c97e-a7c7-4cea-8e4d-ddf622960c5d","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d2657c5e-aaa0-4c43-8cc3-11ef851aaa16","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d2657c5e-aaa0-4c43-8cc3-11ef851aaa16","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d2657c5e-aaa0-4c43-8cc3-11ef851aaa16","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9cf4c97e-a7c7-4cea-8e4d-ddf622960c5d","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d2657c5e-aaa0-4c43-8cc3-11ef851aaa16"}
{"level":"info","ts":"2026-06-11T10:15:13.432Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"d2657c5e-aaa0-4c43-8cc3-11ef851aaa16","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:13.444Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9c635aec-98ca-45e0-86c1-df107f752f03","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:15:13.456Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9c635aec-98ca-45e0-86c1-df107f752f03","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:15:13.456Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"9c635aec-98ca-45e0-86c1-df107f752f03","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1"}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1"}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:14:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:15.432Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"7d7ddc43-6efc-4d9b-a90a-a7ccbcde19c1","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:15.665Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4960722a-7a36-4caa-b98c-0bb068fe58bc","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:15.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4960722a-7a36-4caa-b98c-0bb068fe58bc","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:15:15.666Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4960722a-7a36-4caa-b98c-0bb068fe58bc","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:15:15.666Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4960722a-7a36-4caa-b98c-0bb068fe58bc","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19","10.134.0.26"]}
{"level":"info","ts":"2026-06-11T10:15:15.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"105e9130-421c-4cd2-b297-55fb5167088d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:15.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"105e9130-421c-4cd2-b297-55fb5167088d","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:15.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"105e9130-421c-4cd2-b297-55fb5167088d","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:15.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"105e9130-421c-4cd2-b297-55fb5167088d"}
{"level":"info","ts":"2026-06-11T10:15:15.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"105e9130-421c-4cd2-b297-55fb5167088d","RayCluster":"stop-94xjr","State":""}
{"level":"info","ts":"2026-06-11T10:15:16.582Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43e2451d-0819-4f62-9129-ea2da88ae672","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:15:16.582Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43e2451d-0819-4f62-9129-ea2da88ae672","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:15:16.582Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43e2451d-0819-4f62-9129-ea2da88ae672","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:15:16.583Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43e2451d-0819-4f62-9129-ea2da88ae672","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1"}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1"}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:15:17.172Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"b2023b79-edbf-42ba-9c1a-20a02146e2d1","name":"stop-94xjr","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"674c42ab-4b6c-4193-a385-3d8f1790afdd","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e30c9db9-4562-45a7-8b7c-9917b343284d","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"674c42ab-4b6c-4193-a385-3d8f1790afdd","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"674c42ab-4b6c-4193-a385-3d8f1790afdd","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"674c42ab-4b6c-4193-a385-3d8f1790afdd","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","head service name":"stop-94xjr-head-svc","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","head service URL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","SubmissionMode":"HTTPMode","RayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:15:17.188Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"e44852a8-bb04-458b-b295-a77af8649d11","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:15:17.202Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ef407057-3a53-4831-98ee-4060ab125ae2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:17.202Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ef407057-3a53-4831-98ee-4060ab125ae2","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:17.202Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ef407057-3a53-4831-98ee-4060ab125ae2","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:17.203Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ef407057-3a53-4831-98ee-4060ab125ae2"}
{"level":"info","ts":"2026-06-11T10:15:17.205Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e30c9db9-4562-45a7-8b7c-9917b343284d","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:15:17.211Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ef407057-3a53-4831-98ee-4060ab125ae2","JobId":"stop-dqmvv"}
{"level":"info","ts":"2026-06-11T10:15:17.211Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ef407057-3a53-4831-98ee-4060ab125ae2","rayJob":"stop","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/stop.py\",\"submission_id\":\"stop-dqmvv\"}"}
{"level":"info","ts":"2026-06-11T10:15:17.218Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e30c9db9-4562-45a7-8b7c-9917b343284d","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:15:17.218Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"e30c9db9-4562-45a7-8b7c-9917b343284d","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:18.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3bf31168-821f-44ab-87f3-bd2137f7940c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:18.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3bf31168-821f-44ab-87f3-bd2137f7940c","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:18.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3bf31168-821f-44ab-87f3-bd2137f7940c","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:18.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3bf31168-821f-44ab-87f3-bd2137f7940c"}
{"level":"info","ts":"2026-06-11T10:15:18.952Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3bf31168-821f-44ab-87f3-bd2137f7940c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:18.952Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3bf31168-821f-44ab-87f3-bd2137f7940c","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:15:18.968Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"16fb198c-b206-40b5-ae48-d128672d0595","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:18.969Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"16fb198c-b206-40b5-ae48-d128672d0595","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:18.969Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"16fb198c-b206-40b5-ae48-d128672d0595","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:18.969Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"16fb198c-b206-40b5-ae48-d128672d0595"}
{"level":"info","ts":"2026-06-11T10:15:18.972Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"16fb198c-b206-40b5-ae48-d128672d0595","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:19.188Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde"}
{"level":"info","ts":"2026-06-11T10:15:19.188Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde"}
{"level":"info","ts":"2026-06-11T10:15:19.188Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:19.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:19.189Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"15e32027-c154-476b-9524-b52eed200dde","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:21.968Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ea8d556f-6e92-479e-b132-22faa842cd61","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:21.968Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ea8d556f-6e92-479e-b132-22faa842cd61","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:21.968Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ea8d556f-6e92-479e-b132-22faa842cd61","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:21.968Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ea8d556f-6e92-479e-b132-22faa842cd61"}
{"level":"info","ts":"2026-06-11T10:15:21.971Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ea8d556f-6e92-479e-b132-22faa842cd61","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:21.971Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ea8d556f-6e92-479e-b132-22faa842cd61","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:15:21.991Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6ca67aca-dcbe-4f6d-bf99-9fee05451ee5","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:21.991Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6ca67aca-dcbe-4f6d-bf99-9fee05451ee5","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:21.991Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6ca67aca-dcbe-4f6d-bf99-9fee05451ee5","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:21.992Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6ca67aca-dcbe-4f6d-bf99-9fee05451ee5"}
{"level":"info","ts":"2026-06-11T10:15:21.994Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6ca67aca-dcbe-4f6d-bf99-9fee05451ee5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:22.038Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed848a51-bf57-4556-8e94-4971e54539a0","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:22.038Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed848a51-bf57-4556-8e94-4971e54539a0","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:22.038Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed848a51-bf57-4556-8e94-4971e54539a0","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:22.039Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"ed848a51-bf57-4556-8e94-4971e54539a0","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:24.992Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3c6cd5d6-d23f-452d-83c9-efcab9add35a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:24.992Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3c6cd5d6-d23f-452d-83c9-efcab9add35a","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:24.992Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3c6cd5d6-d23f-452d-83c9-efcab9add35a","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:24.992Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3c6cd5d6-d23f-452d-83c9-efcab9add35a"}
{"level":"info","ts":"2026-06-11T10:15:24.995Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"3c6cd5d6-d23f-452d-83c9-efcab9add35a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:27.996Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"79e05be5-e7eb-4703-8f04-2b31aab05e87","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:27.996Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"79e05be5-e7eb-4703-8f04-2b31aab05e87","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:27.996Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"79e05be5-e7eb-4703-8f04-2b31aab05e87","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:27.996Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"79e05be5-e7eb-4703-8f04-2b31aab05e87"}
{"level":"info","ts":"2026-06-11T10:15:27.999Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"79e05be5-e7eb-4703-8f04-2b31aab05e87","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495"}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495"}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"The Pod status of the worker Pod stop-94xjr-small-group-worker-vk89n is Running, and the Ray container terminated status is &ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-11 10:15:05 +0000 UTC,FinishedAt:2026-06-11 10:15:27 +0000 UTC,ContainerID:cri-o://486c40cebabf1d4ad8ca25832d8948fc5b40a75f912c0c4c9d2673e9ec4f7e5f,}. However, KubeRay will not delete the Pod because its restartPolicy is set to Always and it should be able to restart automatically."}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:28.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:15:28.231Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:15:28.231Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"63b9f2ff-2e2b-4f15-9730-ba0570e9a495","name":"stop-94xjr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:28Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"2635548c-37d8-4c6f-ab78-dd78227096a3","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"f1e299ca-9733-4696-9593-d1e060c48f6e","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"f1e299ca-9733-4696-9593-d1e060c48f6e","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"f1e299ca-9733-4696-9593-d1e060c48f6e","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbd22b67-af15-4206-908c-82d6a61d5a6b","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"2635548c-37d8-4c6f-ab78-dd78227096a3","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"2635548c-37d8-4c6f-ab78-dd78227096a3","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"2635548c-37d8-4c6f-ab78-dd78227096a3","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:28.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"f1e299ca-9733-4696-9593-d1e060c48f6e"}
{"level":"info","ts":"2026-06-11T10:15:28.348Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"f1e299ca-9733-4696-9593-d1e060c48f6e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:28Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:28.360Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbd22b67-af15-4206-908c-82d6a61d5a6b","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:15:28.427Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbd22b67-af15-4206-908c-82d6a61d5a6b","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:15:28.427Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbd22b67-af15-4206-908c-82d6a61d5a6b","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f"}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f"}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:29.230Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"f4bc2ac4-1f83-422a-acf0-729810d0851f","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d"}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d"}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","podName":"stop-94xjr-small-group-worker-vk89n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:30.345Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"9ec1a096-4d4a-4c14-939c-69ce8cea463d","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:31.000Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8e600abd-f5ab-4992-b57a-989e0a9e64a9","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:31.000Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8e600abd-f5ab-4992-b57a-989e0a9e64a9","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:31.000Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8e600abd-f5ab-4992-b57a-989e0a9e64a9","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:31.000Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8e600abd-f5ab-4992-b57a-989e0a9e64a9"}
{"level":"info","ts":"2026-06-11T10:15:31.003Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8e600abd-f5ab-4992-b57a-989e0a9e64a9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:28Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:34.004Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"4673de10-5837-4bdf-b47d-957282c8e33c","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:34.004Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"4673de10-5837-4bdf-b47d-957282c8e33c","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:34.004Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"4673de10-5837-4bdf-b47d-957282c8e33c","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:34.005Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"4673de10-5837-4bdf-b47d-957282c8e33c"}
{"level":"info","ts":"2026-06-11T10:15:34.007Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"4673de10-5837-4bdf-b47d-957282c8e33c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:28Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:37.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"9cf41115-e50b-4763-b604-dd967e7294d2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:37.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"9cf41115-e50b-4763-b604-dd967e7294d2","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:37.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"9cf41115-e50b-4763-b604-dd967e7294d2","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:37.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"9cf41115-e50b-4763-b604-dd967e7294d2"}
{"level":"info","ts":"2026-06-11T10:15:37.011Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"9cf41115-e50b-4763-b604-dd967e7294d2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:28Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:40.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"47e4c2fe-bc6d-4f29-be6a-26328bfa1143","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:40.012Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"47e4c2fe-bc6d-4f29-be6a-26328bfa1143","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:40.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"47e4c2fe-bc6d-4f29-be6a-26328bfa1143","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:40.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"47e4c2fe-bc6d-4f29-be6a-26328bfa1143"}
{"level":"info","ts":"2026-06-11T10:15:40.015Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"47e4c2fe-bc6d-4f29-be6a-26328bfa1143","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:28Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d"}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d"}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","1 head service found":"stop-94xjr-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","Found 1 head Pod":"stop-94xjr-head-r7225","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","head Pod":"stop-94xjr-head-r7225","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-94xjr-head-r7225. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","worker Pod":"stop-94xjr-small-group-worker-vk89n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-94xjr-small-group-worker-vk89n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":1,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:15:40.254Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"de6945b6-4c55-478b-b6b9-347159bb365d","name":"stop-94xjr","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:28Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:40Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:40.268Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"19990de6-65b6-4af4-91dd-898f33057ed2","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:40.268Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbe8a224-4f2c-4090-93db-780581f63c6a","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:40.268Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"499a7bc8-eb1e-47fe-aed5-e3b8b4a4e799","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:40.269Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"499a7bc8-eb1e-47fe-aed5-e3b8b4a4e799","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:40.269Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"19990de6-65b6-4af4-91dd-898f33057ed2","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:40.269Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"19990de6-65b6-4af4-91dd-898f33057ed2","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:40.269Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"499a7bc8-eb1e-47fe-aed5-e3b8b4a4e799","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:40.269Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"19990de6-65b6-4af4-91dd-898f33057ed2","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:40.269Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"499a7bc8-eb1e-47fe-aed5-e3b8b4a4e799"}
{"level":"info","ts":"2026-06-11T10:15:40.271Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"499a7bc8-eb1e-47fe-aed5-e3b8b4a4e799","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:17Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:15:17Z","endTime":"2026-06-11T10:15:40Z"},"jobId":"stop-dqmvv","rayClusterName":"stop-94xjr","dashboardURL":"stop-94xjr-head-svc.test-ns-ccphs.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:14:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:40Z","stateTransitionTimes":{"ready":"2026-06-11T10:15:17Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"stop-94xjr-head-r7225","serviceName":"stop-94xjr-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:15:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:15:17Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:14:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:14:52Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:15:40.272Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"499a7bc8-eb1e-47fe-aed5-e3b8b4a4e799","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-11T10:15:40.285Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6604325b-ec7e-406d-a495-e03fc9e6053a","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:40.285Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"6604325b-ec7e-406d-a495-e03fc9e6053a","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:15:40 +0000 UTC","Now":"2026-06-11T10:15:40.285Z","ShutdownTime":"2026-06-11T10:15:40.000Z"}
{"level":"info","ts":"2026-06-11T10:15:40.291Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbe8a224-4f2c-4090-93db-780581f63c6a","name":"stop-94xjr-head"}
{"level":"info","ts":"2026-06-11T10:15:40.304Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbe8a224-4f2c-4090-93db-780581f63c6a","name":"stop-94xjr-workers"}
{"level":"info","ts":"2026-06-11T10:15:40.304Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"dbe8a224-4f2c-4090-93db-780581f63c6a","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.110Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ff6c0fa3-ff81-4acd-91c6-2270b82263fc","DeletionTimestamp":"2026-06-11 10:15:41 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:15:41.110Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"ff6c0fa3-ff81-4acd-91c6-2270b82263fc","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:15:41.124Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"a31fe528-b62b-4348-a030-fcd2d1325003"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"bd7bbb93-3aa6-41be-b090-9e1b18bbf6e3","rayCluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","namespacedName":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"reconcileID":"fec50e56-b41b-4ad9-a99b-584c8ce8f7a0"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"71553b42-8a28-41e2-b485-745805e135a0"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"bd7bbb93-3aa6-41be-b090-9e1b18bbf6e3","pod":"stop-94xjr-head-r7225","ip":"10.134.0.27"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"bd7bbb93-3aa6-41be-b090-9e1b18bbf6e3","pod":"stop-94xjr-small-group-worker-vk89n","ip":"10.133.0.20"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","namespace":"test-ns-ccphs","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","name":"kuberay-gateway-access","namespace":"test-ns-ccphs","remainingClusters":1}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"bd7bbb93-3aa6-41be-b090-9e1b18bbf6e3","rayCluster":"stop-94xjr","podIPs":["10.134.0.27","10.133.0.20"]}
{"level":"info","ts":"2026-06-11T10:15:41.146Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"161584c0-cd88-4290-a0d3-78b20d6e15f7"}
{"level":"info","ts":"2026-06-11T10:15:41.152Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","name":"test-ns-ccphs-stop-94xjr","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:15:41.157Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","configMap":"kube-rbac-proxy-config-stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.162Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","serviceAccount":"stop-94xjr-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:41.169Z","logger":"raycluster-resource","msg":"default","name":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.169Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-94xjr","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"5d271abd-2ddb-4212-b132-f20eb3f78fd6","namespace":"test-ns-ccphs","clusterName":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"885e4e6c-0233-4f8d-82b9-cc762a21de0c"}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"57a840d7-85e0-4664-9ecf-cdabe6ac73c8"}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"f2cb2906-7653-470b-ab92-f402d9f5feec","cluster":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"4a6222c1-b9b3-45ed-b365-14f904f17c34","namespacedName":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"4a6222c1-b9b3-45ed-b365-14f904f17c34"}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"4a6222c1-b9b3-45ed-b365-14f904f17c34","namespace":"test-ns-ccphs","authClusters":1}
{"level":"info","ts":"2026-06-11T10:15:41.182Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"4a6222c1-b9b3-45ed-b365-14f904f17c34","namespace":"test-ns-ccphs","authClusters":1}
{"level":"info","ts":"2026-06-11T10:15:41.193Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"48d68528-0135-4332-9357-94d5c67bce6a"}
{"level":"info","ts":"2026-06-11T10:15:41.210Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"d82f4fd7-2281-4198-bf2b-5cec2f9e05f1","namespacedName":{"name":"stop-94xjr","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:41.227Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"d82f4fd7-2281-4198-bf2b-5cec2f9e05f1"}
{"level":"info","ts":"2026-06-11T10:15:41.227Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"d82f4fd7-2281-4198-bf2b-5cec2f9e05f1","namespace":"test-ns-ccphs","authClusters":1}
{"level":"info","ts":"2026-06-11T10:15:41.227Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"d82f4fd7-2281-4198-bf2b-5cec2f9e05f1","namespace":"test-ns-ccphs","authClusters":1}
{"level":"info","ts":"2026-06-11T10:15:41.227Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"3121b75f-9654-4751-99f4-3268819ab158"}
{"level":"info","ts":"2026-06-11T10:15:41.247Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"5d271abd-2ddb-4212-b132-f20eb3f78fd6","namespace":"test-ns-ccphs","clusterName":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:15:41.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ecdbb9f-06a7-42f1-8996-90de98336f2e","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:41.376Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ecdbb9f-06a7-42f1-8996-90de98336f2e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:15:41.389Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ecdbb9f-06a7-42f1-8996-90de98336f2e"}
{"level":"info","ts":"2026-06-11T10:15:41.389Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ecdbb9f-06a7-42f1-8996-90de98336f2e","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:15:41.389Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ecdbb9f-06a7-42f1-8996-90de98336f2e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:15:41.389Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ecdbb9f-06a7-42f1-8996-90de98336f2e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:15:41.401Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b504f5bc-89cd-42eb-93b6-5a4267bf6221","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:41.401Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b504f5bc-89cd-42eb-93b6-5a4267bf6221","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.401Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b504f5bc-89cd-42eb-93b6-5a4267bf6221","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.401Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b504f5bc-89cd-42eb-93b6-5a4267bf6221","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.406Z","logger":"raycluster-resource","msg":"default","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.406Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b504f5bc-89cd-42eb-93b6-5a4267bf6221","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"99d9edf6-1cd3-4575-b680-2bb7038e7220","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","mode":"IntegratedOAuth","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b504f5bc-89cd-42eb-93b6-5a4267bf6221"}
{"level":"info","ts":"2026-06-11T10:15:41.416Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b504f5bc-89cd-42eb-93b6-5a4267bf6221","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:41.417Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0c501a8f-1e7e-4c91-8ee7-39c0556f1b46","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:41.417Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0c501a8f-1e7e-4c91-8ee7-39c0556f1b46","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.417Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0c501a8f-1e7e-4c91-8ee7-39c0556f1b46","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.417Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0c501a8f-1e7e-4c91-8ee7-39c0556f1b46"}
{"level":"info","ts":"2026-06-11T10:15:41.417Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0c501a8f-1e7e-4c91-8ee7-39c0556f1b46","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:41.422Z","logger":"raycluster-resource","msg":"default","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.422Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","name":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:15:41.427Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","name":"long-running-sjrp6","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:41.428Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.428Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.433Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"02235763-e610-4d39-89d1-7c59ca472d8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:41.433Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"02235763-e610-4d39-89d1-7c59ca472d8b","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.433Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"02235763-e610-4d39-89d1-7c59ca472d8b","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.433Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"02235763-e610-4d39-89d1-7c59ca472d8b"}
{"level":"info","ts":"2026-06-11T10:15:41.433Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"02235763-e610-4d39-89d1-7c59ca472d8b","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:41.434Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"99d9edf6-1cd3-4575-b680-2bb7038e7220","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.434Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"b44dfd27-4369-41a6-b523-f367ee13ffaf","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.441Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","name":"long-running-sjrp6-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:41.442Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","name":"long-running-sjrp6","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-sjrp6\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-v8w6c","name":"long-running-sjrp6"}}
{"level":"info","ts":"2026-06-11T10:15:41.442Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95"}
{"level":"error","ts":"2026-06-11T10:15:41.442Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0aec9294-357c-4c31-9f51-3c311cb2cb95","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa"}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa"}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:15:41.443Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","name":"long-running-sjrp6","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:41.446Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e"}
{"level":"info","ts":"2026-06-11T10:15:41.446Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:15:41.446Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.449Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","name":"kuberay-gateway-access","namespace":"test-ns-v8w6c","operation":"created"}
{"level":"info","ts":"2026-06-11T10:15:41.452Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"b44dfd27-4369-41a6-b523-f367ee13ffaf","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","name":"test-ns-v8w6c-long-running-sjrp6","operation":"created"}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f22b9a38-2542-4bf6-90cc-e15efa7b7387","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f22b9a38-2542-4bf6-90cc-e15efa7b7387","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f22b9a38-2542-4bf6-90cc-e15efa7b7387","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f22b9a38-2542-4bf6-90cc-e15efa7b7387"}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f22b9a38-2542-4bf6-90cc-e15efa7b7387","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:41.528Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa"}
{"level":"error","ts":"2026-06-11T10:15:41.528Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"f736e3d1-e67c-4a52-ba47-643f45973efa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f"}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f"}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f"}
{"level":"error","ts":"2026-06-11T10:15:41.529Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"7c36e670-d3fc-45ab-a8d1-38d4594baf7f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.533Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"b44dfd27-4369-41a6-b523-f367ee13ffaf","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:41.533Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"b44dfd27-4369-41a6-b523-f367ee13ffaf","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.533Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1948d4f9-adf2-4ff1-a20b-b43cbb12b258","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.535Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","name":"kube-rbac-proxy-config-long-running-sjrp6","operation":"created"}
{"level":"info","ts":"2026-06-11T10:15:41.537Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e"}
{"level":"info","ts":"2026-06-11T10:15:41.537Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:15:41.539Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab"}
{"level":"info","ts":"2026-06-11T10:15:41.539Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab"}
{"level":"info","ts":"2026-06-11T10:15:41.539Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:41.539Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:41.539Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.540Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:41.540Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.540Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.540Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.540Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab"}
{"level":"error","ts":"2026-06-11T10:15:41.540Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a79aecca-52a8-493f-ab3e-43d376fcd0ab","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:15:41.542Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-sjrp6\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.542Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f"}
{"level":"error","ts":"2026-06-11T10:15:41.542Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"41ce2fa5-0eec-4f33-a240-c39863401e0f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-sjrp6\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.542Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2c01c324-3038-4d13-bdc2-8388f9d2ab9b","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.542Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2c01c324-3038-4d13-bdc2-8388f9d2ab9b","mode":"IntegratedOAuth","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.542Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2c01c324-3038-4d13-bdc2-8388f9d2ab9b","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.627Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e"}
{"level":"info","ts":"2026-06-11T10:15:41.627Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","rayCluster":"long-running-sjrp6","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:15:41.627Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-sjrp6\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.627Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e"}
{"level":"error","ts":"2026-06-11T10:15:41.627Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"d500aa03-9ca8-470c-a5a1-626c0acdf37e","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-sjrp6\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f9555798-c409-4bd4-bb7d-e9090c9278ae","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1948d4f9-adf2-4ff1-a20b-b43cbb12b258","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:41.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b"}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b"}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b"}
{"level":"error","ts":"2026-06-11T10:15:41.628Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d6a8d3c6-bcaa-4d04-af4c-c5ff3c74605b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.640Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2c01c324-3038-4d13-bdc2-8388f9d2ab9b","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.640Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f2eb0f2d-f309-46ff-9786-43b336f011f3","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.640Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f2eb0f2d-f309-46ff-9786-43b336f011f3","mode":"IntegratedOAuth","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.640Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f2eb0f2d-f309-46ff-9786-43b336f011f3","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"07dc2e29-ed2c-4895-aa87-bf9811426742","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:41.641Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"07dc2e29-ed2c-4895-aa87-bf9811426742","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.641Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"07dc2e29-ed2c-4895-aa87-bf9811426742","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:41.641Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"07dc2e29-ed2c-4895-aa87-bf9811426742"}
{"level":"info","ts":"2026-06-11T10:15:41.641Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"07dc2e29-ed2c-4895-aa87-bf9811426742","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:41.642Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1948d4f9-adf2-4ff1-a20b-b43cbb12b258","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:41.642Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1948d4f9-adf2-4ff1-a20b-b43cbb12b258","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.642Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"cea101e9-1941-45e3-920a-818d536f3078","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.643Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f9555798-c409-4bd4-bb7d-e9090c9278ae"}
{"level":"error","ts":"2026-06-11T10:15:41.643Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f9555798-c409-4bd4-bb7d-e9090c9278ae","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-sjrp6\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:15:41.643Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f9555798-c409-4bd4-bb7d-e9090c9278ae","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-sjrp6\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.643Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f9555798-c409-4bd4-bb7d-e9090c9278ae"}
{"level":"error","ts":"2026-06-11T10:15:41.643Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f9555798-c409-4bd4-bb7d-e9090c9278ae","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-sjrp6\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.643Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"a6651283-5a78-4484-a107-d360f2fb4dfe","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.643Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"a6651283-5a78-4484-a107-d360f2fb4dfe"}
{"level":"info","ts":"2026-06-11T10:15:41.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6"}
{"level":"info","ts":"2026-06-11T10:15:41.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6"}
{"level":"info","ts":"2026-06-11T10:15:41.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:41.727Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:41.727Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.727Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:41.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.728Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.729Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.729Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6"}
{"level":"error","ts":"2026-06-11T10:15:41.729Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"02afcc15-5a12-4376-9db2-c06c2ffbe3e6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.729Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"61032988-f469-4c96-a354-f18938402b74","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.729Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"61032988-f469-4c96-a354-f18938402b74"}
{"level":"info","ts":"2026-06-11T10:15:41.728Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"f2eb0f2d-f309-46ff-9786-43b336f011f3","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.734Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"cea101e9-1941-45e3-920a-818d536f3078","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:41.743Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"cea101e9-1941-45e3-920a-818d536f3078","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:41.743Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"cea101e9-1941-45e3-920a-818d536f3078","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa"}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa"}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa"}
{"level":"error","ts":"2026-06-11T10:15:41.890Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"00a57a6b-dd96-44f1-a1f4-b4323811c8fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086"}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086"}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086"}
{"level":"error","ts":"2026-06-11T10:15:42.211Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"a5a7b1f5-904c-40bf-a277-b40109734086","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:42.852Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b"}
{"level":"info","ts":"2026-06-11T10:15:42.852Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b"}
{"level":"info","ts":"2026-06-11T10:15:42.852Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:42.852Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","error":"head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"error","ts":"2026-06-11T10:15:42.852Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:42.852Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found"}
{"level":"info","ts":"2026-06-11T10:15:42.853Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:42.853Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:42.853Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-sjrp6","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:42.853Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b"}
{"level":"error","ts":"2026-06-11T10:15:42.853Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dddb4cd0-d816-4589-99af-2b314fc9ae1b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-sjrp6 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:43.015Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-ccphs"},"reconcileID":"8c08885c-7c80-49c1-8766-326d81ee0765"}
{"level":"info","ts":"2026-06-11T10:15:44.133Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f"}
{"level":"info","ts":"2026-06-11T10:15:44.133Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f"}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f"}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f"}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","mode":"IntegratedOAuth","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","cluster":"long-running-sjrp6","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","cluster":"long-running-sjrp6","authType":"IntegratedOAuth","serviceAccount":"long-running-sjrp6-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-sjrp6","ray.io/group":"headgroup","ray.io/identifier":"long-running-sjrp6-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:15:44.134Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","enabled":true}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","name":"long-running-sjrp6-head-8lkz5"}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f"}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","nodeType":"worker","rayStartParams":{"address":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","rayStartCmd":"ray start  --address=long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:44.153Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","enabled":true}
{"level":"info","ts":"2026-06-11T10:15:44.168Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","name":"long-running-sjrp6-small-group-worker-2rbhp"}
{"level":"info","ts":"2026-06-11T10:15:44.168Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.168Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.169Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-sjrp6-head-svc"},"newHead":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"}}
{"level":"info","ts":"2026-06-11T10:15:44.169Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"c2f3bac6-7d2a-4ca1-88a9-f2a5c0e9dc9f","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:44.184Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"174262fb-71de-45cd-8e7b-28fbcb14b606","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"6c820905-6f34-4ffb-8e7e-1ccece455b12","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"79694971-9ba4-4458-a365-279a6202416a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"79694971-9ba4-4458-a365-279a6202416a","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-sjrp6-head-svc"},"newHead":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"}}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"79694971-9ba4-4458-a365-279a6202416a","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"79694971-9ba4-4458-a365-279a6202416a"}
{"level":"info","ts":"2026-06-11T10:15:44.185Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"79694971-9ba4-4458-a365-279a6202416a","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:44.196Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","name":"long-running-sjrp6","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-sjrp6\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-v8w6c","name":"long-running-sjrp6"}}
{"level":"info","ts":"2026-06-11T10:15:44.196Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547"}
{"level":"error","ts":"2026-06-11T10:15:44.196Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0e55b2eb-50a2-47f7-a6a7-41f65ce80547","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-sjrp6\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:44.200Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"42041f53-2caa-4a2a-b780-75cd88717dbd","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"174262fb-71de-45cd-8e7b-28fbcb14b606","rayCluster":"long-running-sjrp6","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"6c820905-6f34-4ffb-8e7e-1ccece455b12","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.227Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.228Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:15:44.228Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"bba32837-1a64-4e2f-bad4-cfb9e982465a","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:44.241Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"6c820905-6f34-4ffb-8e7e-1ccece455b12","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:44.241Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"6c820905-6f34-4ffb-8e7e-1ccece455b12","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.242Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"915fd309-5b63-490e-b5fb-911e8925134c","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.242Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"98d00d88-52a1-4d36-86c6-ba83743a636e","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6ad7357d-3853-4be1-8da7-6a03c98dba6d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6ad7357d-3853-4be1-8da7-6a03c98dba6d","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6ad7357d-3853-4be1-8da7-6a03c98dba6d","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"98d00d88-52a1-4d36-86c6-ba83743a636e","rayCluster":"long-running-sjrp6","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6ad7357d-3853-4be1-8da7-6a03c98dba6d"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6ad7357d-3853-4be1-8da7-6a03c98dba6d","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:44.244Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"2cef5fe9-96cd-4824-9ec6-a359202aebc9","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:44.261Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"915fd309-5b63-490e-b5fb-911e8925134c","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:44.273Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"915fd309-5b63-490e-b5fb-911e8925134c","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:44.273Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"915fd309-5b63-490e-b5fb-911e8925134c","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.401Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"804bb1ba-0ba8-49c0-8160-f100f6b9496e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:44.401Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"804bb1ba-0ba8-49c0-8160-f100f6b9496e","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:44.401Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"804bb1ba-0ba8-49c0-8160-f100f6b9496e","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:44.401Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"804bb1ba-0ba8-49c0-8160-f100f6b9496e"}
{"level":"info","ts":"2026-06-11T10:15:44.401Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"804bb1ba-0ba8-49c0-8160-f100f6b9496e","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:44.632Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"d9a80b01-6cd2-4306-bff4-c8bfdaa1f64d","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:44.642Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088"}
{"level":"info","ts":"2026-06-11T10:15:44.642Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088"}
{"level":"info","ts":"2026-06-11T10:15:44.642Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:44.643Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3fbcc1f6-2a0e-4d98-9ba0-fd64e0d8f088","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:45.296Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0"}
{"level":"info","ts":"2026-06-11T10:15:45.296Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0"}
{"level":"info","ts":"2026-06-11T10:15:45.296Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:45.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:45.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:45.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:45.297Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"dc9c29ea-4850-4e53-b51e-ba10b5de0dd0","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"newHead":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"}}
{"level":"info","ts":"2026-06-11T10:15:45.563Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"96dcb45e-fc40-4aee-9bf6-dd575d155599","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:44Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c5457c11-8e26-4d4e-a0a6-b5b990bc8fca","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"db878384-e5c0-4a92-9d65-c4c23dc98133","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"db878384-e5c0-4a92-9d65-c4c23dc98133","pod":"long-running-sjrp6-head-8lkz5","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"db878384-e5c0-4a92-9d65-c4c23dc98133","pod":"long-running-sjrp6-small-group-worker-2rbhp","ip":"10.133.0.21"}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4ba16adc-3ad8-4dc8-b103-e5fd481178cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4ba16adc-3ad8-4dc8-b103-e5fd481178cb","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4ba16adc-3ad8-4dc8-b103-e5fd481178cb","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4ba16adc-3ad8-4dc8-b103-e5fd481178cb"}
{"level":"info","ts":"2026-06-11T10:15:45.577Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4ba16adc-3ad8-4dc8-b103-e5fd481178cb","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:45.589Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c5457c11-8e26-4d4e-a0a6-b5b990bc8fca","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:45.590Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"db878384-e5c0-4a92-9d65-c4c23dc98133","certificate":"ray-head-cert-long-running-sjrp6","dnsNames":["long-running-sjrp6-head-svc","localhost","long-running-sjrp6-head-svc.test-ns-v8w6c.svc","long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.134.0.28","10.133.0.21","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:45.605Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c5457c11-8e26-4d4e-a0a6-b5b990bc8fca","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:45.605Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c5457c11-8e26-4d4e-a0a6-b5b990bc8fca","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:45.605Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"db878384-e5c0-4a92-9d65-c4c23dc98133","certificate":"ray-worker-cert-long-running-sjrp6","dnsNames":["long-running-sjrp6-worker-svc","localhost","long-running-sjrp6-small-group","long-running-sjrp6-worker-svc.test-ns-v8w6c.svc","long-running-sjrp6-worker-svc.test-ns-v8w6c.svc.cluster.local","*.long-running-sjrp6-worker-svc.test-ns-v8w6c.svc","*.long-running-sjrp6-worker-svc.test-ns-v8w6c.svc.cluster.local","*-worker-*.test-ns-v8w6c.svc","*-worker-*.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.134.0.28","10.133.0.21","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:45.605Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"db878384-e5c0-4a92-9d65-c4c23dc98133","rayCluster":"long-running-sjrp6","podIPs":["10.134.0.28","10.133.0.21"]}
{"level":"info","ts":"2026-06-11T10:15:46.244Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308"}
{"level":"info","ts":"2026-06-11T10:15:46.244Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308"}
{"level":"info","ts":"2026-06-11T10:15:46.244Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:46.245Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"3b239eee-1349-4a6c-923d-232192624308","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961"}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961"}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:46.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.264Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:46.264Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3c5c374e-91c7-41db-8c6d-fe2ad4c75961","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c"}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c"}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:46.270Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"6eefe062-19ce-4bcf-b984-c48186a0a55c","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:46.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664"}
{"level":"info","ts":"2026-06-11T10:15:46.289Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664"}
{"level":"info","ts":"2026-06-11T10:15:46.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:46.289Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:46.290Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9dc3908d-d7a1-4e22-b636-d4cef9da4664","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5"}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5"}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-s9zgk-small-group-worker-gqm59. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:46.293Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"07e965cf-cb6d-44c5-9990-8f5276c5def5","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:46.844Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad"}
{"level":"info","ts":"2026-06-11T10:15:46.844Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad"}
{"level":"info","ts":"2026-06-11T10:15:46.844Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:46.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:46.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:46.844Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","worker Pod":"fail-s9zgk-small-group-worker-gqm59","shouldDelete":true,"reason":"The worker Pod fail-s9zgk-small-group-worker-gqm59 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-11T10:15:46.848Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:46.848Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-11T10:15:46.848Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:15:46.848Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","name":"fail-s9zgk","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:14:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:46Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"042f08bc-ba15-453b-8a77-7f86d42a7bf5","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad"}
{"level":"error","ts":"2026-06-11T10:15:46.864Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"38431cee-ae98-4afa-9f35-ed2d96c0cfad","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d89aaebe-242f-48fc-ba82-76b1f94716aa","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"042f08bc-ba15-453b-8a77-7f86d42a7bf5","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"042f08bc-ba15-453b-8a77-7f86d42a7bf5","pod":"fail-s9zgk-small-group-worker-gqm59","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"042f08bc-ba15-453b-8a77-7f86d42a7bf5","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19","10.134.0.26"]}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"25df3126-74dc-47a9-aed0-84b58736c885","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:46.864Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"25df3126-74dc-47a9-aed0-84b58736c885","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:15:46.864Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd"}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd"}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:46.869Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a07c1c5b-d2da-4789-aef8-d25e7253efcd","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:46.870Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d89aaebe-242f-48fc-ba82-76b1f94716aa","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:47.401Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f88c053-75d6-427b-8161-64cd3e7b21d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:47.401Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f88c053-75d6-427b-8161-64cd3e7b21d8","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:47.401Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f88c053-75d6-427b-8161-64cd3e7b21d8","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:47.402Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f88c053-75d6-427b-8161-64cd3e7b21d8"}
{"level":"info","ts":"2026-06-11T10:15:47.402Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f88c053-75d6-427b-8161-64cd3e7b21d8","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98"}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98"}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98","podName":"fail-s9zgk-small-group-worker-gqm59","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:47.572Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"aaf532fb-7a03-45f0-959a-23f31662ba98","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91"}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91"}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91"}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.575Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.576Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:47.579Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.579Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:47.579Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}]}
{"level":"info","ts":"2026-06-11T10:15:47.580Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","name":"fail-s9zgk","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:46Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:47.592Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91"}
{"level":"error","ts":"2026-06-11T10:15:47.592Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"20a732c9-b793-462c-b248-e60b445eef91","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.592Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"117a3aba-711a-403e-b6fe-0dc119a695a1","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:47.592Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ba3f9a8b-7ae7-4a5b-b9c0-00ca99d62e9b","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:47.592Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"165871c1-ac0b-4baa-8f45-bdbfdde1bcca","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:47.592Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"165871c1-ac0b-4baa-8f45-bdbfdde1bcca","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:15:47.592Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"info","ts":"2026-06-11T10:15:47.592Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ba3f9a8b-7ae7-4a5b-b9c0-00ca99d62e9b","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"117a3aba-711a-403e-b6fe-0dc119a695a1","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236"}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.597Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:47.601Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.601Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:47.601Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236"}
{"level":"error","ts":"2026-06-11T10:15:47.601Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"0776cc6d-b589-46e1-9fe4-5be41f718236","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.606Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ba3f9a8b-7ae7-4a5b-b9c0-00ca99d62e9b","certificate":"ray-head-cert-fail-s9zgk","dnsNames":["fail-s9zgk-head-svc","localhost","fail-s9zgk-head-svc.test-ns-ccphs.svc","fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29"}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29"}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29"}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.612Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:47.616Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.616Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:47.616Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29"}
{"level":"error","ts":"2026-06-11T10:15:47.616Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a61c82ad-2d53-4ee9-bf08-a19d71a9fc29","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.616Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ba3f9a8b-7ae7-4a5b-b9c0-00ca99d62e9b","certificate":"ray-worker-cert-fail-s9zgk","dnsNames":["fail-s9zgk-worker-svc","localhost","fail-s9zgk-small-group","fail-s9zgk-worker-svc.test-ns-ccphs.svc","fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*-worker-*.test-ns-ccphs.svc","*-worker-*.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["10.133.0.19","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:47.617Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"ba3f9a8b-7ae7-4a5b-b9c0-00ca99d62e9b","rayCluster":"fail-s9zgk","podIPs":["10.133.0.19"]}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2"}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2"}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2"}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.637Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:47.640Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.641Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:47.641Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2"}
{"level":"error","ts":"2026-06-11T10:15:47.641Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"2c5e4b3f-ca61-43ec-b6de-8bb24d12a6e2","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c"}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.682Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:47.686Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.686Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:47.686Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c"}
{"level":"error","ts":"2026-06-11T10:15:47.686Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"554486a7-92bb-4706-bbbb-a21e54ec7b1c","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61"}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.768Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:47.771Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.771Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:47.771Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61"}
{"level":"error","ts":"2026-06-11T10:15:47.771Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"5dca3591-7c68-48c9-b311-acada04fba61","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-s9zgk-head-bqwsq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914"}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:47.932Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:47.936Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:47.936Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:47.936Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914"}
{"level":"error","ts":"2026-06-11T10:15:47.936Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"9cd54371-e4ed-47c9-ae38-80c0dd3c8914","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.134Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287"}
{"level":"info","ts":"2026-06-11T10:15:48.134Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287"}
{"level":"info","ts":"2026-06-11T10:15:48.134Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:48.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","Found 1 head Pod":"fail-s9zgk-head-bqwsq","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-11 10:14:17 +0000 UTC,FinishedAt:2026-06-11 10:15:47 +0000 UTC,ContainerID:cri-o://f12920789fc37c20c34725cec9707c5d3c3aa5f044526a86527e2d8534021b6e,}"}
{"level":"info","ts":"2026-06-11T10:15:48.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","head Pod":"fail-s9zgk-head-bqwsq","shouldDelete":true,"reason":"The head Pod fail-s9zgk-head-bqwsq status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-11T10:15:48.138Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod fail-s9zgk-head-bqwsq status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.138Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","error":"The head Pod fail-s9zgk-head-bqwsq status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-11T10:15:48.138Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:48Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}]}
{"level":"info","ts":"2026-06-11T10:15:48.138Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","name":"fail-s9zgk","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:47Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:14:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:48Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:48Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:48.154Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287"}
{"level":"error","ts":"2026-06-11T10:15:48.154Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"7b23c821-755a-462d-a6b8-489c7e939287","error":"The head Pod fail-s9zgk-head-bqwsq status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.155Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d7cd2eeb-f932-46ab-bf4e-a9b21a0724cb","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:48.155Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3b8cd389-7ffa-4edc-b0eb-69492f6c6a7c","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:48.155Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3b8cd389-7ffa-4edc-b0eb-69492f6c6a7c","pod":"fail-s9zgk-head-bqwsq","ip":"10.133.0.19"}
{"level":"info","ts":"2026-06-11T10:15:48.155Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0159814d-e1ce-476d-ad62-a076e2d51371","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:48.155Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"0159814d-e1ce-476d-ad62-a076e2d51371","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:15:48.155Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"info","ts":"2026-06-11T10:15:48.155Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3b8cd389-7ffa-4edc-b0eb-69492f6c6a7c"}
{"level":"info","ts":"2026-06-11T10:15:48.233Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d7cd2eeb-f932-46ab-bf4e-a9b21a0724cb","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef"}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:48.257Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:48.260Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.260Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:48.260Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef"}
{"level":"error","ts":"2026-06-11T10:15:48.260Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"d75a12ee-4e7f-4cef-97e4-f5eae6c6b6ef","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463"}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","nodeType":"worker","rayStartParams":{"address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","rayStartCmd":"ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:15:48.314Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:48.318Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.318Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:48.318Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463"}
{"level":"error","ts":"2026-06-11T10:15:48.318Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"e80729ec-8d91-4db2-b641-55518a528463","error":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","mode":"IntegratedOAuth","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","cluster":"fail-s9zgk","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","cluster":"fail-s9zgk","authType":"IntegratedOAuth","serviceAccount":"fail-s9zgk-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-s9zgk","ray.io/group":"headgroup","ray.io/identifier":"fail-s9zgk-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:15:48.323Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:48.327Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-s9zgk-head-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.327Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","error":"FailedCreateHeadPod\npods \"fail-s9zgk-head-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:48.327Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:48.327Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:48.327Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:48.327Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"newHead":{"serviceName":"fail-s9zgk-head-svc"}}
{"level":"info","ts":"2026-06-11T10:15:48.327Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","name":"fail-s9zgk","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:48Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.19","serviceIP":"10.133.0.19","podName":"fail-s9zgk-head-bqwsq","serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:48Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-s9zgk-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:48Z","stateTransitionTimes":{"ready":"2026-06-11T10:14:47Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-s9zgk-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:48Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:14:47Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:12:45Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:12:45Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:15:47Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-s9zgk-head-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:48.340Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e"}
{"level":"error","ts":"2026-06-11T10:15:48.340Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"df5038ee-e140-413e-956b-ec8e8a6f3f6e","error":"FailedCreateHeadPod\npods \"fail-s9zgk-head-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:48.340Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d205d5bb-acdb-4bd1-b28a-3314e9f998ef","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:48.340Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"df3ae384-8db7-40c2-98d1-c40644047103","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:48.340Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cc2f42b7-8071-4c67-9704-c9f279f1cfc2","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:48.340Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"cc2f42b7-8071-4c67-9704-c9f279f1cfc2","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:15:48.340Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"info","ts":"2026-06-11T10:15:48.346Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"d205d5bb-acdb-4bd1-b28a-3314e9f998ef","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:48.354Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"df3ae384-8db7-40c2-98d1-c40644047103","certificate":"ray-head-cert-fail-s9zgk","dnsNames":["fail-s9zgk-head-svc","localhost","fail-s9zgk-head-svc.test-ns-ccphs.svc","fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:48.369Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"df3ae384-8db7-40c2-98d1-c40644047103","certificate":"ray-worker-cert-fail-s9zgk","dnsNames":["fail-s9zgk-worker-svc","localhost","fail-s9zgk-small-group","fail-s9zgk-worker-svc.test-ns-ccphs.svc","fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc","*.fail-s9zgk-worker-svc.test-ns-ccphs.svc.cluster.local","*-worker-*.test-ns-ccphs.svc","*-worker-*.test-ns-ccphs.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:15:48.369Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"df3ae384-8db7-40c2-98d1-c40644047103"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","1 head service found":"fail-s9zgk-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","mode":"IntegratedOAuth","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","cluster":"fail-s9zgk","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","cluster":"fail-s9zgk","authType":"IntegratedOAuth","serviceAccount":"fail-s9zgk-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-s9zgk","ray.io/group":"headgroup","ray.io/identifier":"fail-s9zgk-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-s9zgk-head-svc.test-ns-ccphs.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:15:49.541Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","enabled":true}
{"level":"error","ts":"2026-06-11T10:15:49.546Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-s9zgk-head-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:49.546Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","error":"FailedCreateHeadPod\npods \"fail-s9zgk-head-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:15:49.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:49.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:49.546Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","filter labels":["test-ns-ccphs",{"ray.io/cluster":"fail-s9zgk","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:15:49.546Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb"}
{"level":"error","ts":"2026-06-11T10:15:49.546Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"a911e86e-d9a5-4975-82c1-7392e87f8ccb","error":"FailedCreateHeadPod\npods \"fail-s9zgk-head-\" is forbidden: unable to create new content in namespace test-ns-ccphs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:50.402Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ce6167a2-7016-4013-8890-207fe48aba79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:50.402Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ce6167a2-7016-4013-8890-207fe48aba79","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:50.402Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ce6167a2-7016-4013-8890-207fe48aba79","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:50.402Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ce6167a2-7016-4013-8890-207fe48aba79"}
{"level":"info","ts":"2026-06-11T10:15:50.402Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ce6167a2-7016-4013-8890-207fe48aba79","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:50.586Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e"}
{"level":"info","ts":"2026-06-11T10:15:50.586Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e"}
{"level":"info","ts":"2026-06-11T10:15:50.586Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:50.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:50.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:50.586Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","podName":"long-running-sjrp6-head-8lkz5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:50.587Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"76c22de2-eb1c-487e-bb38-2d29729c5c9e","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"86b99836-092e-4078-bf14-6fb45caa595a","rayCluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"reconcileID":"3ac723b3-2657-4ee2-8381-c83218eebdd2"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"3476d931-4697-4486-a214-62870c63071b"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"86b99836-092e-4078-bf14-6fb45caa595a"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","namespace":"test-ns-ccphs","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"14444b26-0a56-474c-98cd-9582553f26ef","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:51.494Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"14444b26-0a56-474c-98cd-9582553f26ef","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:15:51.494Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"info","ts":"2026-06-11T10:15:51.499Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","name":"kuberay-gateway-access","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:15:51.506Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","name":"test-ns-ccphs-fail-s9zgk","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:15:51.512Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","serviceAccount":"fail-s9zgk-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:15:51.519Z","logger":"raycluster-resource","msg":"default","name":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.519Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-s9zgk","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:15:51.545Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"c3ed2a34-8913-468c-b2b4-6440d21a63da","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.545Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:51.545Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.545Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e","cluster":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.545Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e","namespace":"test-ns-ccphs","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:15:51.546Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:15:51.546Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"6fb1e1ef-e131-47b8-9e4e-60d15658ef5d"}
{"level":"info","ts":"2026-06-11T10:15:51.546Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4c296cbf-9195-4d8b-a62b-0e0b24f05875"}
{"level":"info","ts":"2026-06-11T10:15:51.546Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"573c7af8-08d5-4e9d-ad6b-c2e2659251ee","namespace":"test-ns-ccphs","clusterName":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.546Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"207d8e05-cbde-4788-9098-4f2d4e1d9624","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-11T10:15:51.546Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"207d8e05-cbde-4788-9098-4f2d4e1d9624","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:14:51 +0000 UTC","Now":"2026-06-11T10:15:51.546Z","ShutdownTime":"2026-06-11T10:14:51.000Z"}
{"level":"error","ts":"2026-06-11T10:15:51.549Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"573c7af8-08d5-4e9d-ad6b-c2e2659251ee","issuer":"ray-selfsigned-issuer-fail-s9zgk","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-s9zgk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:15:51.550Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e","error":"rayclusters.ray.io \"fail-s9zgk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:51.550Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e"}
{"level":"error","ts":"2026-06-11T10:15:51.550Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"a2759faa-a851-4a0f-9db0-d4d5fcc3623e","error":"rayclusters.ray.io \"fail-s9zgk\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:51.550Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"afa1fa36-84ca-4ee5-be26-543d886bd7e0","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:51.550Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"afa1fa36-84ca-4ee5-be26-543d886bd7e0"}
{"level":"info","ts":"2026-06-11T10:15:51.550Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"afa1fa36-84ca-4ee5-be26-543d886bd7e0","namespace":"test-ns-ccphs","authClusters":0}
{"level":"info","ts":"2026-06-11T10:15:51.550Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"afa1fa36-84ca-4ee5-be26-543d886bd7e0","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:15:51.551Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"9ac9cb6b-7367-49a0-84d7-3f0e566e0d8a","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:51.552Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"9ac9cb6b-7367-49a0-84d7-3f0e566e0d8a"}
{"level":"info","ts":"2026-06-11T10:15:51.552Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"9ac9cb6b-7367-49a0-84d7-3f0e566e0d8a","namespace":"test-ns-ccphs","authClusters":0}
{"level":"info","ts":"2026-06-11T10:15:51.552Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"9ac9cb6b-7367-49a0-84d7-3f0e566e0d8a","namespace":"test-ns-ccphs"}
{"level":"info","ts":"2026-06-11T10:15:51.555Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"882047fe-df6a-49d0-a5e2-3b700d858980","namespacedName":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"}}
{"level":"info","ts":"2026-06-11T10:15:51.556Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"882047fe-df6a-49d0-a5e2-3b700d858980"}
{"level":"info","ts":"2026-06-11T10:15:51.556Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"882047fe-df6a-49d0-a5e2-3b700d858980","namespace":"test-ns-ccphs","authClusters":0}
{"level":"info","ts":"2026-06-11T10:15:51.556Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"882047fe-df6a-49d0-a5e2-3b700d858980","namespace":"test-ns-ccphs"}
{"level":"error","ts":"2026-06-11T10:15:51.556Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"573c7af8-08d5-4e9d-ad6b-c2e2659251ee","certificate":"ray-worker-cert-fail-s9zgk","error":"certificates.cert-manager.io \"ray-worker-cert-fail-s9zgk\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:51.574Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"573c7af8-08d5-4e9d-ad6b-c2e2659251ee","errorCount":2}
{"level":"info","ts":"2026-06-11T10:15:51.574Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"573c7af8-08d5-4e9d-ad6b-c2e2659251ee"}
{"level":"error","ts":"2026-06-11T10:15:51.574Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"573c7af8-08d5-4e9d-ad6b-c2e2659251ee","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-s9zgk\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:15:51.579Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4cae24df-8412-4206-83ad-eb74fdd33169","namespace":"test-ns-ccphs","clusterName":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.579Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-s9zgk","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"fail-s9zgk","reconcileID":"4cae24df-8412-4206-83ad-eb74fdd33169","namespace":"test-ns-ccphs","clusterName":"fail-s9zgk"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","podName":"long-running-sjrp6-small-group-worker-2rbhp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","podName":"long-running-sjrp6-small-group-worker-2rbhp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:15:51.593Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:15:51.594Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"0b575f62-03be-4044-a491-c13f434e8da0","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"43372670-b48d-4c9f-80c1-b60d2367e5ed","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"958cfcb7-c43d-4b5f-b0b1-1cb3d731b55e","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"43372670-b48d-4c9f-80c1-b60d2367e5ed","pod":"long-running-sjrp6-head-8lkz5","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"43372670-b48d-4c9f-80c1-b60d2367e5ed","pod":"long-running-sjrp6-small-group-worker-2rbhp","ip":"10.133.0.21"}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"43372670-b48d-4c9f-80c1-b60d2367e5ed","rayCluster":"long-running-sjrp6","podIPs":["10.134.0.28","10.133.0.21"]}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"1d66f8b0-67c5-4064-afde-5546247a7585","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"1d66f8b0-67c5-4064-afde-5546247a7585","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:51.607Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"1d66f8b0-67c5-4064-afde-5546247a7585","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:51.608Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"1d66f8b0-67c5-4064-afde-5546247a7585"}
{"level":"info","ts":"2026-06-11T10:15:51.608Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"1d66f8b0-67c5-4064-afde-5546247a7585","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:51.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"958cfcb7-c43d-4b5f-b0b1-1cb3d731b55e","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:51.638Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"958cfcb7-c43d-4b5f-b0b1-1cb3d731b55e","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:51.638Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"958cfcb7-c43d-4b5f-b0b1-1cb3d731b55e","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:51.959Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"2a3a1dbd-4802-4ddd-b4c1-2a35189945e3","DeletionTimestamp":"2026-06-11 10:15:51 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:15:51.959Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"2a3a1dbd-4802-4ddd-b4c1-2a35189945e3","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:15:51.972Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"841fdee5-1325-4b66-9555-a86a6d7e68db","DeletionTimestamp":"2026-06-11 10:15:51 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:15:51.972Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"841fdee5-1325-4b66-9555-a86a6d7e68db","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:15:51.988Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"0a10694d-b729-462e-89b5-e6a1c8ee293b"}
{"level":"info","ts":"2026-06-11T10:15:51.988Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"7fc80c0d-d0bb-4728-b240-f75f97997cef"}
{"level":"info","ts":"2026-06-11T10:15:53.403Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f1b66ad-2ae2-4196-80db-380dd3818cad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:53.403Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f1b66ad-2ae2-4196-80db-380dd3818cad","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:53.403Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f1b66ad-2ae2-4196-80db-380dd3818cad","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:53.403Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f1b66ad-2ae2-4196-80db-380dd3818cad"}
{"level":"info","ts":"2026-06-11T10:15:53.403Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6f1b66ad-2ae2-4196-80db-380dd3818cad","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:53.610Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886"}
{"level":"info","ts":"2026-06-11T10:15:53.610Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886"}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:53.611Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"11c08c98-8eda-4abd-854a-eacf9bca8886","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:54.972Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-ccphs"},"reconcileID":"0158a427-4962-4306-b8cd-c76dc794ee27"}
{"level":"info","ts":"2026-06-11T10:15:54.989Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-ccphs"},"reconcileID":"021dc889-668b-4e3f-877b-8eb5a7c2a502"}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b"}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b"}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:55.353Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"82c66956-727a-4f6d-bc18-9db5cf05054b","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392"}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392"}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:56.353Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:56.354Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:15:56.354Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ce028f73-eee9-43de-b236-2f9177240392","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:15:56.372Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"37a38698-abb9-4964-98d7-5b0af0ddaf93","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:56.372Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"45de4fce-662c-47c9-afd1-b935067513a5","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"a6c6c936-f8b6-4c39-9283-684e35ca36bd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"a6c6c936-f8b6-4c39-9283-684e35ca36bd","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"a6c6c936-f8b6-4c39-9283-684e35ca36bd","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"37a38698-abb9-4964-98d7-5b0af0ddaf93","pod":"long-running-sjrp6-head-8lkz5","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"37a38698-abb9-4964-98d7-5b0af0ddaf93","pod":"long-running-sjrp6-small-group-worker-2rbhp","ip":"10.133.0.21"}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"37a38698-abb9-4964-98d7-5b0af0ddaf93","rayCluster":"long-running-sjrp6","podIPs":["10.134.0.28","10.133.0.21"]}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"a6c6c936-f8b6-4c39-9283-684e35ca36bd"}
{"level":"info","ts":"2026-06-11T10:15:56.373Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"a6c6c936-f8b6-4c39-9283-684e35ca36bd","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:56.388Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"45de4fce-662c-47c9-afd1-b935067513a5","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:15:56.402Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"45de4fce-662c-47c9-afd1-b935067513a5","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:15:56.402Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"45de4fce-662c-47c9-afd1-b935067513a5","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:56.404Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ebc057f8-4655-4ddd-b350-a96b84698490","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:56.404Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ebc057f8-4655-4ddd-b350-a96b84698490","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:56.404Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ebc057f8-4655-4ddd-b350-a96b84698490","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:56.404Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ebc057f8-4655-4ddd-b350-a96b84698490"}
{"level":"info","ts":"2026-06-11T10:15:56.404Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"ebc057f8-4655-4ddd-b350-a96b84698490","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:15:58.373Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77"}
{"level":"info","ts":"2026-06-11T10:15:58.373Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77"}
{"level":"info","ts":"2026-06-11T10:15:58.373Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","podName":"long-running-sjrp6-head-8lkz5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:15:58.374Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"831746e1-31a2-4170-9843-2dca87fb7c77","seconds":300}
{"level":"info","ts":"2026-06-11T10:15:59.405Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"eab3c9b2-46d1-40fb-be91-16d984e0906b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:15:59.405Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"eab3c9b2-46d1-40fb-be91-16d984e0906b","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:15:59.405Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"eab3c9b2-46d1-40fb-be91-16d984e0906b","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:15:59.405Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"eab3c9b2-46d1-40fb-be91-16d984e0906b"}
{"level":"info","ts":"2026-06-11T10:15:59.405Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"eab3c9b2-46d1-40fb-be91-16d984e0906b","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:16:02.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"46c9e7a9-78d2-4af4-bc02-67d6a424007c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:02.406Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"46c9e7a9-78d2-4af4-bc02-67d6a424007c","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:02.406Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"46c9e7a9-78d2-4af4-bc02-67d6a424007c","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:02.406Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"46c9e7a9-78d2-4af4-bc02-67d6a424007c"}
{"level":"info","ts":"2026-06-11T10:16:02.406Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"46c9e7a9-78d2-4af4-bc02-67d6a424007c","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612"}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612"}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","podName":"long-running-sjrp6-small-group-worker-2rbhp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","podName":"long-running-sjrp6-small-group-worker-2rbhp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:16:02.628Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"139013ed-6d32-48f9-b389-07dbec5fc612","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:15:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:02.642Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"31b22063-0268-4d43-97e7-1f2017bcd4b4","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:02.642Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c7557edb-77d4-4aaf-863e-fd7cd00bd1e4","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:02.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"206a45fc-3124-4be8-9b3b-3be1a26800d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:02.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"206a45fc-3124-4be8-9b3b-3be1a26800d8","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:02.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"206a45fc-3124-4be8-9b3b-3be1a26800d8","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:02.643Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c7557edb-77d4-4aaf-863e-fd7cd00bd1e4","pod":"long-running-sjrp6-head-8lkz5","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-11T10:16:02.643Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c7557edb-77d4-4aaf-863e-fd7cd00bd1e4","pod":"long-running-sjrp6-small-group-worker-2rbhp","ip":"10.133.0.21"}
{"level":"info","ts":"2026-06-11T10:16:02.643Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"206a45fc-3124-4be8-9b3b-3be1a26800d8"}
{"level":"info","ts":"2026-06-11T10:16:02.643Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"206a45fc-3124-4be8-9b3b-3be1a26800d8","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:16:02.643Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"c7557edb-77d4-4aaf-863e-fd7cd00bd1e4","rayCluster":"long-running-sjrp6","podIPs":["10.134.0.28","10.133.0.21"]}
{"level":"info","ts":"2026-06-11T10:16:02.656Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"31b22063-0268-4d43-97e7-1f2017bcd4b4","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:16:02.732Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"31b22063-0268-4d43-97e7-1f2017bcd4b4","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:16:02.732Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"31b22063-0268-4d43-97e7-1f2017bcd4b4","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3"}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3"}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","podName":"long-running-sjrp6-small-group-worker-2rbhp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","podName":"long-running-sjrp6-small-group-worker-2rbhp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:15:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:04.642Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"12326a62-7846-459d-ad8e-85cd7dcf50c3","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:05.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7524fb95-1748-43f9-9cf2-3182ef48a5ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:05.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7524fb95-1748-43f9-9cf2-3182ef48a5ec","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:05.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7524fb95-1748-43f9-9cf2-3182ef48a5ec","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:05.408Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7524fb95-1748-43f9-9cf2-3182ef48a5ec"}
{"level":"info","ts":"2026-06-11T10:16:05.408Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7524fb95-1748-43f9-9cf2-3182ef48a5ec","RayCluster":"long-running-sjrp6","State":""}
{"level":"info","ts":"2026-06-11T10:16:07.371Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9"}
{"level":"info","ts":"2026-06-11T10:16:07.371Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9"}
{"level":"info","ts":"2026-06-11T10:16:07.371Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","1 head service found":"long-running-sjrp6-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:07.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","Found 1 head Pod":"long-running-sjrp6-head-8lkz5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","head Pod":"long-running-sjrp6-head-8lkz5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-sjrp6-head-8lkz5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","worker Pod":"long-running-sjrp6-small-group-worker-2rbhp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-sjrp6-small-group-worker-2rbhp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:16:07.372Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"704b32e8-04fa-42b5-9c6c-88bf2826ebb9","name":"long-running-sjrp6","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:07Z","stateTransitionTimes":{"ready":"2026-06-11T10:16:07Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"long-running-sjrp6-head-8lkz5","serviceName":"long-running-sjrp6-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:02Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:16:07Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:15:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:15:41Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"5398f128-4b02-46a6-ade1-5b82b8f16fdc","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e39af3b-1fee-4d40-bea8-d503fdea2dbd","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"5398f128-4b02-46a6-ade1-5b82b8f16fdc","pod":"long-running-sjrp6-head-8lkz5","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"5398f128-4b02-46a6-ade1-5b82b8f16fdc","pod":"long-running-sjrp6-small-group-worker-2rbhp","ip":"10.133.0.21"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"5398f128-4b02-46a6-ade1-5b82b8f16fdc","rayCluster":"long-running-sjrp6","podIPs":["10.134.0.28","10.133.0.21"]}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","head service name":"long-running-sjrp6-head-svc","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","head service URL":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383"}
{"level":"info","ts":"2026-06-11T10:16:07.385Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","command":["if","!","ray","job","status","--address","http://long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","long-running-q7l82",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","--no-wait","--submission-id","long-running-q7l82","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","--follow","long-running-q7l82"]}
{"level":"info","ts":"2026-06-11T10:16:07.398Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-11T10:16:07.398Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","SubmissionMode":"K8sJobMode","RayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.398Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","dashboardURL":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:07.398Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"574c266e-73b4-4af9-a957-c1289bdc3383","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:16:07.405Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e39af3b-1fee-4d40-bea8-d503fdea2dbd","name":"long-running-sjrp6-head"}
{"level":"info","ts":"2026-06-11T10:16:07.409Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.409Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.409Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.410Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32"}
{"level":"info","ts":"2026-06-11T10:16:07.410Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","head service name":"long-running-sjrp6-head-svc","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:07.410Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","head service URL":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:16:07.410Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-11T10:16:07.410Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","SubmissionMode":"K8sJobMode","RayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.410Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","dashboardURL":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:07.410Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e39af3b-1fee-4d40-bea8-d503fdea2dbd","name":"long-running-sjrp6-workers"}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e39af3b-1fee-4d40-bea8-d503fdea2dbd","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32"}
{"level":"error","ts":"2026-06-11T10:16:07.419Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"7696c102-6429-47b5-8348-d69dca6bff32","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9d3ab3e0-d7ae-45f8-bbf1-d98532af804c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9d3ab3e0-d7ae-45f8-bbf1-d98532af804c","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9d3ab3e0-d7ae-45f8-bbf1-d98532af804c","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.419Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9d3ab3e0-d7ae-45f8-bbf1-d98532af804c"}
{"level":"error","ts":"2026-06-11T10:16:07.427Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9d3ab3e0-d7ae-45f8-bbf1-d98532af804c","JobId":"long-running-q7l82","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.427Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9d3ab3e0-d7ae-45f8-bbf1-d98532af804c"}
{"level":"error","ts":"2026-06-11T10:16:07.427Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9d3ab3e0-d7ae-45f8-bbf1-d98532af804c","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.427Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"228d6979-6003-4c86-8381-fd35dd2cc00e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.427Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"228d6979-6003-4c86-8381-fd35dd2cc00e","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.427Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"228d6979-6003-4c86-8381-fd35dd2cc00e","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.427Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"228d6979-6003-4c86-8381-fd35dd2cc00e"}
{"level":"error","ts":"2026-06-11T10:16:07.432Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"228d6979-6003-4c86-8381-fd35dd2cc00e","JobId":"long-running-q7l82","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.432Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"228d6979-6003-4c86-8381-fd35dd2cc00e"}
{"level":"error","ts":"2026-06-11T10:16:07.432Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"228d6979-6003-4c86-8381-fd35dd2cc00e","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.438Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75fc8ada-1546-4902-a55b-32e00e399638","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.438Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75fc8ada-1546-4902-a55b-32e00e399638","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.438Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75fc8ada-1546-4902-a55b-32e00e399638","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.438Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75fc8ada-1546-4902-a55b-32e00e399638"}
{"level":"error","ts":"2026-06-11T10:16:07.443Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75fc8ada-1546-4902-a55b-32e00e399638","JobId":"long-running-q7l82","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.443Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75fc8ada-1546-4902-a55b-32e00e399638"}
{"level":"error","ts":"2026-06-11T10:16:07.443Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75fc8ada-1546-4902-a55b-32e00e399638","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.483Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9f7f1c25-0540-4dd3-be8a-827eed3e1823","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.483Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9f7f1c25-0540-4dd3-be8a-827eed3e1823","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.483Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9f7f1c25-0540-4dd3-be8a-827eed3e1823","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.483Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9f7f1c25-0540-4dd3-be8a-827eed3e1823"}
{"level":"error","ts":"2026-06-11T10:16:07.489Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9f7f1c25-0540-4dd3-be8a-827eed3e1823","JobId":"long-running-q7l82","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.489Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9f7f1c25-0540-4dd3-be8a-827eed3e1823"}
{"level":"error","ts":"2026-06-11T10:16:07.489Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9f7f1c25-0540-4dd3-be8a-827eed3e1823","error":"Job long-running-q7l82 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.545Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c50dae97-7177-4efb-8f0a-8c0d7d4a118c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.545Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c50dae97-7177-4efb-8f0a-8c0d7d4a118c","oldStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:16:07.545Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c50dae97-7177-4efb-8f0a-8c0d7d4a118c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","dashboardURL":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","dashboardURL":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:07.545Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c50dae97-7177-4efb-8f0a-8c0d7d4a118c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-06-11T10:16:07.558Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b4c77b03-6f0a-47a1-8345-a0f00bf4b58e","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.569Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b4c77b03-6f0a-47a1-8345-a0f00bf4b58e","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.569Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b4c77b03-6f0a-47a1-8345-a0f00bf4b58e","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"fc4f6cf2-813a-4e02-a692-8afcd1bcea84","rayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"reconcileID":"ba67680b-abd0-421e-81ae-f055b7600cad"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"680f7dbd-7efc-4143-92b6-537f2ce6ef02"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"fc4f6cf2-813a-4e02-a692-8afcd1bcea84","pod":"long-running-sjrp6-head-8lkz5","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"fc4f6cf2-813a-4e02-a692-8afcd1bcea84","pod":"long-running-sjrp6-small-group-worker-2rbhp","ip":"10.133.0.21"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","namespace":"test-ns-v8w6c","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:16:07.570Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"fc4f6cf2-813a-4e02-a692-8afcd1bcea84","rayCluster":"long-running-sjrp6","podIPs":["10.134.0.28","10.133.0.21"]}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"The associated submitter Kubernetes Job for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b4c77b03-6f0a-47a1-8345-a0f00bf4b58e","Submitter K8s Job":"long-running"}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b4c77b03-6f0a-47a1-8345-a0f00bf4b58e","isJobDeleted":false}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"b4c77b03-6f0a-47a1-8345-a0f00bf4b58e"}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","name":"kuberay-gateway-access","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5081644e-03c7-40ec-ac92-191ee4f6ac58","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5081644e-03c7-40ec-ac92-191ee4f6ac58","RayCluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5081644e-03c7-40ec-ac92-191ee4f6ac58","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5081644e-03c7-40ec-ac92-191ee4f6ac58","Kubernetes Job":""}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5081644e-03c7-40ec-ac92-191ee4f6ac58","isJobDeleted":true}
{"level":"info","ts":"2026-06-11T10:16:07.575Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5081644e-03c7-40ec-ac92-191ee4f6ac58"}
{"level":"info","ts":"2026-06-11T10:16:07.581Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","name":"test-ns-v8w6c-long-running-sjrp6","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:16:07.589Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","configMap":"kube-rbac-proxy-config-long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.598Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","serviceAccount":"long-running-sjrp6-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:07.605Z","logger":"raycluster-resource","msg":"default","name":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.605Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:07.616Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"431d67b2-ed20-4526-bf72-abcfe07e9539","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.616Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.616Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.616Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4","cluster":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.616Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4","namespace":"test-ns-v8w6c","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:16:07.616Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:16:07.616Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"0db20bc7-bdee-4898-82a2-b7c796802fd3","namespace":"test-ns-v8w6c","clusterName":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75744a80-4b5b-4a56-a5c7-fca15d2fca0c","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.617Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75744a80-4b5b-4a56-a5c7-fca15d2fca0c","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.617Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75744a80-4b5b-4a56-a5c7-fca15d2fca0c","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:16:07.617Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75744a80-4b5b-4a56-a5c7-fca15d2fca0c","Kubernetes Job":""}
{"level":"info","ts":"2026-06-11T10:16:07.617Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75744a80-4b5b-4a56-a5c7-fca15d2fca0c","isJobDeleted":true}
{"level":"info","ts":"2026-06-11T10:16:07.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75744a80-4b5b-4a56-a5c7-fca15d2fca0c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-q7l82","rayClusterName":"long-running-sjrp6","dashboardURL":"long-running-sjrp6-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:07.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"75744a80-4b5b-4a56-a5c7-fca15d2fca0c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"854d8587-7a68-486c-95a4-e0489d198d1c"}
{"level":"error","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4","error":"rayclusters.ray.io \"long-running-sjrp6\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4"}
{"level":"error","ts":"2026-06-11T10:16:07.627Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1f5227df-fdc3-4208-b32e-8b9eaa9517f4","error":"rayclusters.ray.io \"long-running-sjrp6\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"8ff7d081-15ec-4fba-b7b4-1cecdea78cad","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"8ff7d081-15ec-4fba-b7b4-1cecdea78cad"}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"8ff7d081-15ec-4fba-b7b4-1cecdea78cad","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"8ff7d081-15ec-4fba-b7b4-1cecdea78cad","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"810e7b92-c429-4269-abc8-00f0048f327c"}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1e141361-cacf-4606-a3fb-b7fd5784abd1","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1e141361-cacf-4606-a3fb-b7fd5784abd1"}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1e141361-cacf-4606-a3fb-b7fd5784abd1","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:16:07.627Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"1e141361-cacf-4606-a3fb-b7fd5784abd1","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:07.633Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e16fc1a-8634-4d01-a890-5fc822060160","namespacedName":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:07.633Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e16fc1a-8634-4d01-a890-5fc822060160"}
{"level":"info","ts":"2026-06-11T10:16:07.633Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e16fc1a-8634-4d01-a890-5fc822060160","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:16:07.633Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"2e16fc1a-8634-4d01-a890-5fc822060160","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:07.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"127224cc-5f5e-477b-bb67-edbfe03b88c9","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:07.647Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"99408eb7-b058-4e6f-8a7d-9ee0168db96e"}
{"level":"error","ts":"2026-06-11T10:16:07.647Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"0db20bc7-bdee-4898-82a2-b7c796802fd3","certificate":"ray-head-cert-long-running-sjrp6","error":"certificates.cert-manager.io \"ray-head-cert-long-running-sjrp6\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.685Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"0db20bc7-bdee-4898-82a2-b7c796802fd3","errorCount":1}
{"level":"info","ts":"2026-06-11T10:16:07.685Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"0db20bc7-bdee-4898-82a2-b7c796802fd3"}
{"level":"error","ts":"2026-06-11T10:16:07.685Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"0db20bc7-bdee-4898-82a2-b7c796802fd3","error":"certificates.cert-manager.io \"ray-head-cert-long-running-sjrp6\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:07.691Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"fe2b991b-de8c-4126-94fe-9b3fe999473b","namespace":"test-ns-v8w6c","clusterName":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:07.727Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-sjrp6","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-sjrp6","reconcileID":"fe2b991b-de8c-4126-94fe-9b3fe999473b","namespace":"test-ns-v8w6c","clusterName":"long-running-sjrp6"}
{"level":"info","ts":"2026-06-11T10:16:08.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0a90f224-49f2-4c54-8ec4-1d735e9fae84","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:08.579Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0a90f224-49f2-4c54-8ec4-1d735e9fae84"}
{"level":"info","ts":"2026-06-11T10:16:08.579Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0a90f224-49f2-4c54-8ec4-1d735e9fae84","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:08.579Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"0a90f224-49f2-4c54-8ec4-1d735e9fae84","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-06-11T10:16:08.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5a341c0c-5da5-4039-b2c9-fe8989b81600","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:08.591Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5a341c0c-5da5-4039-b2c9-fe8989b81600"}
{"level":"info","ts":"2026-06-11T10:16:08.591Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5a341c0c-5da5-4039-b2c9-fe8989b81600","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:16:08.591Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5a341c0c-5da5-4039-b2c9-fe8989b81600","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-11T10:15:41Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-g4jns","rayClusterName":"long-running-g6v6m","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:08Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:08.591Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5a341c0c-5da5-4039-b2c9-fe8989b81600","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:16:08.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"62e2f466-dbab-4046-b305-5036f3466edb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:08.606Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"62e2f466-dbab-4046-b305-5036f3466edb","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.606Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"62e2f466-dbab-4046-b305-5036f3466edb","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.606Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"62e2f466-dbab-4046-b305-5036f3466edb","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.613Z","logger":"raycluster-resource","msg":"default","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.613Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"77b17605-fcbd-40c2-8a99-db2cb292bf1d","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"77b17605-fcbd-40c2-8a99-db2cb292bf1d","mode":"IntegratedOAuth","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"77b17605-fcbd-40c2-8a99-db2cb292bf1d","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"77b17605-fcbd-40c2-8a99-db2cb292bf1d","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"62e2f466-dbab-4046-b305-5036f3466edb","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.650Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"849e1cd5-38dd-46da-9443-9a1e2b0e0a15","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.650Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"62e2f466-dbab-4046-b305-5036f3466edb"}
{"level":"info","ts":"2026-06-11T10:16:08.650Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"62e2f466-dbab-4046-b305-5036f3466edb","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:08.650Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c1a9653b-388c-40fa-affd-3bd9ccdb2efd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:08.650Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c1a9653b-388c-40fa-affd-3bd9ccdb2efd","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.650Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c1a9653b-388c-40fa-affd-3bd9ccdb2efd","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.650Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c1a9653b-388c-40fa-affd-3bd9ccdb2efd"}
{"level":"info","ts":"2026-06-11T10:16:08.651Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"c1a9653b-388c-40fa-affd-3bd9ccdb2efd","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","name":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","serviceSelectors":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/identifier":"long-running-g6v6m-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","error":"unable to find head service. cluster name long-running-g6v6m, filter labels [test-ns-v8w6c map[ray.io/cluster:long-running-g6v6m ray.io/identifier:long-running-g6v6m-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc"}
{"level":"error","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e3da9245-8fe7-4297-9328-2f4047e2cacc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.658Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8"}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8"}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:16:08.659Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","name":"long-running-g6v6m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:08.661Z","logger":"raycluster-resource","msg":"default","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.661Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:08.670Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.670Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8"}
{"level":"error","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"41061b46-c507-4f43-88e2-c21d123c1de8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4"}
{"level":"info","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4"}
{"level":"info","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.682Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","name":"long-running-g6v6m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"67fd3505-31f9-435c-bb5f-96dc41c72a0f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"67fd3505-31f9-435c-bb5f-96dc41c72a0f","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"67fd3505-31f9-435c-bb5f-96dc41c72a0f","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"849e1cd5-38dd-46da-9443-9a1e2b0e0a15","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"67fd3505-31f9-435c-bb5f-96dc41c72a0f"}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"0eeb7d66-d9ec-47a3-a752-bc6ba7e719a8","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.683Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"67fd3505-31f9-435c-bb5f-96dc41c72a0f","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:08.687Z","logger":"raycluster-resource","msg":"default","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.687Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:08.728Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509"}
{"level":"info","ts":"2026-06-11T10:16:08.741Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:16:08.741Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.734Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","name":"long-running-g6v6m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-g6v6m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-v8w6c","name":"long-running-g6v6m"}}
{"level":"info","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4"}
{"level":"error","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"5f0f493e-20e9-4a38-846b-6daa3c6a2ea4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9"}
{"level":"info","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9"}
{"level":"info","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.742Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:08.743Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.743Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.743Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.743Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9"}
{"level":"error","ts":"2026-06-11T10:16:08.743Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"eac01a3e-a3e7-44b7-ace2-f2d6c2056eb9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:16:08.749Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"77b17605-fcbd-40c2-8a99-db2cb292bf1d","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-g6v6m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.749Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"77b17605-fcbd-40c2-8a99-db2cb292bf1d"}
{"level":"error","ts":"2026-06-11T10:16:08.749Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"77b17605-fcbd-40c2-8a99-db2cb292bf1d","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-g6v6m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.749Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.749Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","mode":"IntegratedOAuth","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.749Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.749Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.751Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"0eeb7d66-d9ec-47a3-a752-bc6ba7e719a8","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:08.753Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.757Z","logger":"raycluster-resource","msg":"default","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.757Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38"}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38"}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38"}
{"level":"error","ts":"2026-06-11T10:16:08.762Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"388a7338-5f75-4df4-a563-54221015fa38","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.763Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509"}
{"level":"info","ts":"2026-06-11T10:16:08.763Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:16:08.768Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"0eeb7d66-d9ec-47a3-a752-bc6ba7e719a8","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:08.768Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"0eeb7d66-d9ec-47a3-a752-bc6ba7e719a8","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509"}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5770686c-ca99-4667-a0bb-ca7dc0e45c5d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5770686c-ca99-4667-a0bb-ca7dc0e45c5d","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5770686c-ca99-4667-a0bb-ca7dc0e45c5d","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","rayCluster":"long-running-g6v6m","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5770686c-ca99-4667-a0bb-ca7dc0e45c5d"}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5770686c-ca99-4667-a0bb-ca7dc0e45c5d","RayCluster":"long-running-g6v6m","State":""}
{"level":"error","ts":"2026-06-11T10:16:08.844Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-g6v6m\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509"}
{"level":"error","ts":"2026-06-11T10:16:08.844Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c377b9a1-90fb-4565-9914-c9ad5fe94509","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-g6v6m\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.844Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7b74eadf-91d3-4b19-98f3-3a4b5a7a4c3c","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.845Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"9d4b14d7-b0f0-431f-9af0-c80aec55b02a","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.845Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"9d4b14d7-b0f0-431f-9af0-c80aec55b02a"}
{"level":"info","ts":"2026-06-11T10:16:08.845Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d"}
{"level":"info","ts":"2026-06-11T10:16:08.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d"}
{"level":"info","ts":"2026-06-11T10:16:08.943Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:08.943Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:08.943Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.943Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:08.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.944Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:08.944Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d"}
{"level":"error","ts":"2026-06-11T10:16:08.944Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9597a55a-cbba-43c2-bffa-ba8c85eef04d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.851Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"3ae78423-b6cb-4c25-83bf-bb8330cc6a42","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.944Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"3ae78423-b6cb-4c25-83bf-bb8330cc6a42"}
{"level":"info","ts":"2026-06-11T10:16:08.930Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","name":"long-running-g6v6m-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:08.953Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","name":"kuberay-gateway-access","namespace":"test-ns-v8w6c","operation":"created"}
{"level":"info","ts":"2026-06-11T10:16:08.958Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7b74eadf-91d3-4b19-98f3-3a4b5a7a4c3c","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:08.962Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","name":"test-ns-v8w6c-long-running-g6v6m","operation":"created"}
{"level":"info","ts":"2026-06-11T10:16:08.966Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","name":"kube-rbac-proxy-config-long-running-g6v6m","operation":"created"}
{"level":"info","ts":"2026-06-11T10:16:08.969Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7b74eadf-91d3-4b19-98f3-3a4b5a7a4c3c","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:08.969Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7b74eadf-91d3-4b19-98f3-3a4b5a7a4c3c","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e51f196b-c38c-42db-a583-8904008bab90","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e7637434-b567-4cdb-81d3-1cf9daf527e3","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e7637434-b567-4cdb-81d3-1cf9daf527e3","mode":"IntegratedOAuth","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e7637434-b567-4cdb-81d3-1cf9daf527e3","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"9ee421da-0852-4563-9820-6ff506c5aa5a","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"d6dbf259-fe9a-41ba-a71f-731ff508d637","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4c47d63b-21aa-4c39-8d49-6ced28020f55","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4c47d63b-21aa-4c39-8d49-6ced28020f55","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"d6dbf259-fe9a-41ba-a71f-731ff508d637"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4c47d63b-21aa-4c39-8d49-6ced28020f55","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4c47d63b-21aa-4c39-8d49-6ced28020f55"}
{"level":"info","ts":"2026-06-11T10:16:08.979Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"4c47d63b-21aa-4c39-8d49-6ced28020f55","RayCluster":"long-running-g6v6m","State":""}
{"level":"error","ts":"2026-06-11T10:16:08.987Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e7637434-b567-4cdb-81d3-1cf9daf527e3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-g6v6m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.987Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e7637434-b567-4cdb-81d3-1cf9daf527e3"}
{"level":"error","ts":"2026-06-11T10:16:08.987Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e7637434-b567-4cdb-81d3-1cf9daf527e3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-g6v6m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:08.991Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"9ee421da-0852-4563-9820-6ff506c5aa5a","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:09.027Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"8137b512-ae43-4d7a-91b7-f445283d6839","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:09.027Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"8137b512-ae43-4d7a-91b7-f445283d6839","mode":"IntegratedOAuth","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:09.027Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"8137b512-ae43-4d7a-91b7-f445283d6839","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:09.031Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"9ee421da-0852-4563-9820-6ff506c5aa5a","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:09.031Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"9ee421da-0852-4563-9820-6ff506c5aa5a","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:09.036Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"8137b512-ae43-4d7a-91b7-f445283d6839","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:09.104Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10"}
{"level":"info","ts":"2026-06-11T10:16:09.104Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10"}
{"level":"info","ts":"2026-06-11T10:16:09.104Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:09.104Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:09.104Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:09.104Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:09.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:09.105Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:09.105Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:09.105Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10"}
{"level":"error","ts":"2026-06-11T10:16:09.105Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"40759f7e-254b-4c85-b56c-6e26cede7c10","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7"}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7"}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7"}
{"level":"error","ts":"2026-06-11T10:16:09.426Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8f92573a-0770-4a31-a1b9-55d8f8fb32b7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9"}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9"}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","error":"head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"error","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found"}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"long-running-g6v6m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9"}
{"level":"error","ts":"2026-06-11T10:16:10.067Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"867fdb4c-7773-4b9d-b028-7ccb3dbda4b9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-g6v6m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:10.576Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"55bc692a-19fb-4307-8689-ee2a8b5047e0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:10.576Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"55bc692a-19fb-4307-8689-ee2a8b5047e0","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:10.576Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"55bc692a-19fb-4307-8689-ee2a8b5047e0","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:10.576Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"55bc692a-19fb-4307-8689-ee2a8b5047e0"}
{"level":"info","ts":"2026-06-11T10:16:10.576Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"55bc692a-19fb-4307-8689-ee2a8b5047e0","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","mode":"IntegratedOAuth","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","cluster":"long-running-g6v6m","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","cluster":"long-running-g6v6m","authType":"IntegratedOAuth","serviceAccount":"long-running-g6v6m-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-g6v6m","ray.io/group":"headgroup","ray.io/identifier":"long-running-g6v6m-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:16:11.348Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","enabled":true}
{"level":"info","ts":"2026-06-11T10:16:11.372Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","name":"long-running-g6v6m-head-9tfk7"}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc"}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","nodeType":"worker","rayStartParams":{"address":"long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","rayStartCmd":"ray start  --address=long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:16:11.373Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","enabled":true}
{"level":"info","ts":"2026-06-11T10:16:11.393Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","name":"long-running-g6v6m-small-group-worker-z454z"}
{"level":"info","ts":"2026-06-11T10:16:11.393Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.393Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-g6v6m-head-svc"},"newHead":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"}}
{"level":"info","ts":"2026-06-11T10:16:11.393Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"3b3a6998-9a20-4ed3-b009-64e8128bf9dc","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c25e855c-c0e4-4c36-a911-69221a4f153a","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5439e2af-7bff-42ad-8fd2-6073655025b9","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"099bb202-4d6d-4113-8486-015a64b3046c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"099bb202-4d6d-4113-8486-015a64b3046c","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5439e2af-7bff-42ad-8fd2-6073655025b9","rayCluster":"long-running-g6v6m","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"099bb202-4d6d-4113-8486-015a64b3046c","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"099bb202-4d6d-4113-8486-015a64b3046c"}
{"level":"info","ts":"2026-06-11T10:16:11.411Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"099bb202-4d6d-4113-8486-015a64b3046c","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:11.412Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"1a18795e-6d13-4c0f-88bf-17c3bbb21cd8","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:16:11.416Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"94540cc7-e9b3-4229-8c50-670baa765905","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:11.433Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c25e855c-c0e4-4c36-a911-69221a4f153a","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562"}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562"}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.434Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c4679fdf-10b0-4bfc-9afd-fcaad0980e98","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6d19dcc0-3003-4d30-b9ab-799b67be2d6c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c4679fdf-10b0-4bfc-9afd-fcaad0980e98","rayCluster":"long-running-g6v6m","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6d19dcc0-3003-4d30-b9ab-799b67be2d6c","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6d19dcc0-3003-4d30-b9ab-799b67be2d6c","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6d19dcc0-3003-4d30-b9ab-799b67be2d6c"}
{"level":"info","ts":"2026-06-11T10:16:11.435Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6d19dcc0-3003-4d30-b9ab-799b67be2d6c","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:11.527Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","name":"long-running-g6v6m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-g6v6m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-v8w6c","name":"long-running-g6v6m"}}
{"level":"info","ts":"2026-06-11T10:16:11.527Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562"}
{"level":"info","ts":"2026-06-11T10:16:11.527Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c25e855c-c0e4-4c36-a911-69221a4f153a","name":"long-running-g6v6m-workers"}
{"level":"error","ts":"2026-06-11T10:16:11.527Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"a89bd86b-03ca-42e9-958a-5cd88dc28562","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-g6v6m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c25e855c-c0e4-4c36-a911-69221a4f153a","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a522a914-82ee-4c27-a687-abe5f372eff4","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.528Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.529Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:11.529Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"9330fbac-ffef-405a-b83e-da54a621fd7c","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:11.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04"}
{"level":"info","ts":"2026-06-11T10:16:11.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04"}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.629Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:11.629Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"0e9586fe-6ca3-4d94-94d9-8646dd0e6b04","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:11.633Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a522a914-82ee-4c27-a687-abe5f372eff4","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:11.739Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a522a914-82ee-4c27-a687-abe5f372eff4","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:11.739Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a522a914-82ee-4c27-a687-abe5f372eff4","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:11.832Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"981eb9a3-83ff-432d-b2ae-9972ecbc35e8","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f"}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f"}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:11.859Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:11.859Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"80ca5288-e9bd-460b-a915-6746caf47d5f","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"newHead":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"}}
{"level":"info","ts":"2026-06-11T10:16:12.423Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"37f2333d-a38b-4f09-ad78-697f37220e88","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"26ba2374-14cb-42fc-b5a2-a76e3d764b01","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"eb7f4a97-cb71-42d6-8879-31e9515487e3","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"41f5a1e6-234e-4f8c-aa2f-4541efedb293","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"41f5a1e6-234e-4f8c-aa2f-4541efedb293","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"41f5a1e6-234e-4f8c-aa2f-4541efedb293","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"eb7f4a97-cb71-42d6-8879-31e9515487e3","pod":"long-running-g6v6m-head-9tfk7","ip":"10.133.0.23"}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"41f5a1e6-234e-4f8c-aa2f-4541efedb293"}
{"level":"info","ts":"2026-06-11T10:16:12.437Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"41f5a1e6-234e-4f8c-aa2f-4541efedb293","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:12.450Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"eb7f4a97-cb71-42d6-8879-31e9515487e3","certificate":"ray-head-cert-long-running-g6v6m","dnsNames":["long-running-g6v6m-head-svc","localhost","long-running-g6v6m-head-svc.test-ns-v8w6c.svc","long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.23","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:16:12.451Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"26ba2374-14cb-42fc-b5a2-a76e3d764b01","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:12.468Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"eb7f4a97-cb71-42d6-8879-31e9515487e3","certificate":"ray-worker-cert-long-running-g6v6m","dnsNames":["long-running-g6v6m-worker-svc","localhost","long-running-g6v6m-small-group","long-running-g6v6m-worker-svc.test-ns-v8w6c.svc","long-running-g6v6m-worker-svc.test-ns-v8w6c.svc.cluster.local","*.long-running-g6v6m-worker-svc.test-ns-v8w6c.svc","*.long-running-g6v6m-worker-svc.test-ns-v8w6c.svc.cluster.local","*-worker-*.test-ns-v8w6c.svc","*-worker-*.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.23","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:16:12.468Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"eb7f4a97-cb71-42d6-8879-31e9515487e3"}
{"level":"info","ts":"2026-06-11T10:16:12.469Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"26ba2374-14cb-42fc-b5a2-a76e3d764b01","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:12.469Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"26ba2374-14cb-42fc-b5a2-a76e3d764b01","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:12.689Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:12.690Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:12.690Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"de76bfc2-6df6-4ead-a137-8fcb0e69345c","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:13.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f088acf1-8486-4f43-9abe-cb698772928d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:13.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f088acf1-8486-4f43-9abe-cb698772928d","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:13.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f088acf1-8486-4f43-9abe-cb698772928d","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:13.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f088acf1-8486-4f43-9abe-cb698772928d"}
{"level":"info","ts":"2026-06-11T10:16:13.577Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f088acf1-8486-4f43-9abe-cb698772928d","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:14.436Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36"}
{"level":"info","ts":"2026-06-11T10:16:14.436Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36"}
{"level":"info","ts":"2026-06-11T10:16:14.436Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:14.437Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcad076b-1687-4e4a-a2ef-be19593a1b36","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:16.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"191eaebc-b2b1-478b-abfd-4d4a136a4ad4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:16.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"191eaebc-b2b1-478b-abfd-4d4a136a4ad4","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:16.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"191eaebc-b2b1-478b-abfd-4d4a136a4ad4","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:16.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"191eaebc-b2b1-478b-abfd-4d4a136a4ad4"}
{"level":"info","ts":"2026-06-11T10:16:16.578Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"191eaebc-b2b1-478b-abfd-4d4a136a4ad4","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:16.583Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"d434bc90-7015-4c07-a5a8-b60cee00cfe2","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:16:16.583Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"d434bc90-7015-4c07-a5a8-b60cee00cfe2","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:16:16.583Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"d434bc90-7015-4c07-a5a8-b60cee00cfe2","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:16:16.583Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"d434bc90-7015-4c07-a5a8-b60cee00cfe2","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:16:17.443Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6"}
{"level":"info","ts":"2026-06-11T10:16:17.443Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6"}
{"level":"info","ts":"2026-06-11T10:16:17.443Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","podName":"long-running-g6v6m-head-9tfk7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:17.444Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ff030dd7-e6f9-4314-b0dd-675738a0a2a6","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989"}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989"}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:16:18.451Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"895c5e46-82ff-4a8d-91fe-63e8c5707989","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:18.463Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"91e5face-f6d9-4132-92c3-cf661f7ebef5","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:18.463Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fab0c1ce-423e-4658-bbe7-1752edfaf544","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:18.463Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e5dc5edd-da2f-4e9b-9291-8e10469ec9e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:18.463Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e5dc5edd-da2f-4e9b-9291-8e10469ec9e7","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:18.463Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e5dc5edd-da2f-4e9b-9291-8e10469ec9e7","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:18.463Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fab0c1ce-423e-4658-bbe7-1752edfaf544","pod":"long-running-g6v6m-head-9tfk7","ip":"10.133.0.23"}
{"level":"info","ts":"2026-06-11T10:16:18.463Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fab0c1ce-423e-4658-bbe7-1752edfaf544","pod":"long-running-g6v6m-small-group-worker-z454z","ip":"10.134.0.29"}
{"level":"info","ts":"2026-06-11T10:16:18.464Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e5dc5edd-da2f-4e9b-9291-8e10469ec9e7"}
{"level":"info","ts":"2026-06-11T10:16:18.464Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e5dc5edd-da2f-4e9b-9291-8e10469ec9e7","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:18.474Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"91e5face-f6d9-4132-92c3-cf661f7ebef5","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:18.481Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fab0c1ce-423e-4658-bbe7-1752edfaf544","certificate":"ray-head-cert-long-running-g6v6m","dnsNames":["long-running-g6v6m-head-svc","localhost","long-running-g6v6m-head-svc.test-ns-v8w6c.svc","long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.23","10.134.0.29","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:16:18.488Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"91e5face-f6d9-4132-92c3-cf661f7ebef5","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:18.488Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"91e5face-f6d9-4132-92c3-cf661f7ebef5","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:18.501Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fab0c1ce-423e-4658-bbe7-1752edfaf544","certificate":"ray-worker-cert-long-running-g6v6m","dnsNames":["long-running-g6v6m-worker-svc","localhost","long-running-g6v6m-small-group","long-running-g6v6m-worker-svc.test-ns-v8w6c.svc","long-running-g6v6m-worker-svc.test-ns-v8w6c.svc.cluster.local","*.long-running-g6v6m-worker-svc.test-ns-v8w6c.svc","*.long-running-g6v6m-worker-svc.test-ns-v8w6c.svc.cluster.local","*-worker-*.test-ns-v8w6c.svc","*-worker-*.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.23","10.134.0.29","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:16:18.501Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fab0c1ce-423e-4658-bbe7-1752edfaf544","rayCluster":"long-running-g6v6m","podIPs":["10.133.0.23","10.134.0.29"]}
{"level":"info","ts":"2026-06-11T10:16:19.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5bd5c77b-9f99-4786-a7a6-4cc4f15410cc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:19.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5bd5c77b-9f99-4786-a7a6-4cc4f15410cc","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:19.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5bd5c77b-9f99-4786-a7a6-4cc4f15410cc","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:19.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5bd5c77b-9f99-4786-a7a6-4cc4f15410cc"}
{"level":"info","ts":"2026-06-11T10:16:19.579Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5bd5c77b-9f99-4786-a7a6-4cc4f15410cc","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93"}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93"}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:20.464Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"28f09c5a-1280-49db-9765-381e65bd8c93","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36"}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36"}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:21.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:21.730Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:21.730Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:21.730Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"6f0258dd-3791-42dc-91c2-45c2ec115a36","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:22.039Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"286d14a2-e363-4eb4-8093-dfb760a94b0b","namespace":"test-ns-ccphs","clusterName":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:16:22.039Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-94xjr","namespace":"test-ns-ccphs"},"namespace":"test-ns-ccphs","name":"stop-94xjr","reconcileID":"286d14a2-e363-4eb4-8093-dfb760a94b0b","namespace":"test-ns-ccphs","clusterName":"stop-94xjr"}
{"level":"info","ts":"2026-06-11T10:16:22.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e89d360c-7a7b-46a5-b98a-721d86768865","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:22.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e89d360c-7a7b-46a5-b98a-721d86768865","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:22.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e89d360c-7a7b-46a5-b98a-721d86768865","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:22.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e89d360c-7a7b-46a5-b98a-721d86768865"}
{"level":"info","ts":"2026-06-11T10:16:22.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"e89d360c-7a7b-46a5-b98a-721d86768865","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:22.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f"}
{"level":"info","ts":"2026-06-11T10:16:22.732Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f"}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:16:22.733Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"d7df6649-62ac-42c1-ae29-98807a32572f","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a9a9e875-d196-413e-a3ea-ffb48515e2b3","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"6a25fb5a-ac2f-4d33-8ff9-719153265f5f","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"6a25fb5a-ac2f-4d33-8ff9-719153265f5f","pod":"long-running-g6v6m-head-9tfk7","ip":"10.133.0.23"}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"6a25fb5a-ac2f-4d33-8ff9-719153265f5f","pod":"long-running-g6v6m-small-group-worker-z454z","ip":"10.134.0.29"}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5cacf9ca-b6b2-40ad-995b-ded8b97288b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5cacf9ca-b6b2-40ad-995b-ded8b97288b1","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"6a25fb5a-ac2f-4d33-8ff9-719153265f5f","rayCluster":"long-running-g6v6m","podIPs":["10.133.0.23","10.134.0.29"]}
{"level":"info","ts":"2026-06-11T10:16:22.746Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5cacf9ca-b6b2-40ad-995b-ded8b97288b1","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:22.747Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5cacf9ca-b6b2-40ad-995b-ded8b97288b1"}
{"level":"info","ts":"2026-06-11T10:16:22.747Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"5cacf9ca-b6b2-40ad-995b-ded8b97288b1","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:22.757Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a9a9e875-d196-413e-a3ea-ffb48515e2b3","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:22.766Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a9a9e875-d196-413e-a3ea-ffb48515e2b3","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:22.766Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"a9a9e875-d196-413e-a3ea-ffb48515e2b3","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350"}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350"}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","podName":"long-running-g6v6m-head-9tfk7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:24.747Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"fcf900b1-a8f2-435a-896f-344eae0ef350","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:25.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ff68cd2-a033-4a09-9fba-de7f1698b527","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:25.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ff68cd2-a033-4a09-9fba-de7f1698b527","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:25.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ff68cd2-a033-4a09-9fba-de7f1698b527","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:25.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ff68cd2-a033-4a09-9fba-de7f1698b527"}
{"level":"info","ts":"2026-06-11T10:16:25.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9ff68cd2-a033-4a09-9fba-de7f1698b527","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:28.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2d62cd8a-0dfa-4c71-bf4d-672b42da9928","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:28.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2d62cd8a-0dfa-4c71-bf4d-672b42da9928","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:28.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2d62cd8a-0dfa-4c71-bf4d-672b42da9928","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:28.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2d62cd8a-0dfa-4c71-bf4d-672b42da9928"}
{"level":"info","ts":"2026-06-11T10:16:28.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2d62cd8a-0dfa-4c71-bf4d-672b42da9928","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593"}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593"}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:16:29.490Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"8db8b32d-431f-4cd9-9409-d29dca81d593","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:29.503Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6cf497a-a868-47ff-98e4-b5fd84bdfba6","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:29.503Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c47ed64e-30fd-4403-89c9-92ef086c345e","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:29.503Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"39380c00-7ee6-4a41-aae1-51d071cb4d77","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:29.504Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"39380c00-7ee6-4a41-aae1-51d071cb4d77","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:29.504Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c47ed64e-30fd-4403-89c9-92ef086c345e","pod":"long-running-g6v6m-head-9tfk7","ip":"10.133.0.23"}
{"level":"info","ts":"2026-06-11T10:16:29.504Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c47ed64e-30fd-4403-89c9-92ef086c345e","pod":"long-running-g6v6m-small-group-worker-z454z","ip":"10.134.0.29"}
{"level":"info","ts":"2026-06-11T10:16:29.504Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"39380c00-7ee6-4a41-aae1-51d071cb4d77","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:29.504Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"c47ed64e-30fd-4403-89c9-92ef086c345e","rayCluster":"long-running-g6v6m","podIPs":["10.133.0.23","10.134.0.29"]}
{"level":"info","ts":"2026-06-11T10:16:29.504Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"39380c00-7ee6-4a41-aae1-51d071cb4d77"}
{"level":"info","ts":"2026-06-11T10:16:29.504Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"39380c00-7ee6-4a41-aae1-51d071cb4d77","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:29.518Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6cf497a-a868-47ff-98e4-b5fd84bdfba6","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:29.530Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6cf497a-a868-47ff-98e4-b5fd84bdfba6","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:29.530Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6cf497a-a868-47ff-98e4-b5fd84bdfba6","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:31.503Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09"}
{"level":"info","ts":"2026-06-11T10:16:31.503Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09"}
{"level":"info","ts":"2026-06-11T10:16:31.503Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:31.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:31.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:31.503Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","podName":"long-running-g6v6m-small-group-worker-z454z","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:11Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:31.504Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"ddf5055b-054f-4a86-97e2-e463bef47b09","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:31.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"df6290a3-91f6-4d23-91e3-ec3406df2a19","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:31.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"df6290a3-91f6-4d23-91e3-ec3406df2a19","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:31.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"df6290a3-91f6-4d23-91e3-ec3406df2a19","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:31.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"df6290a3-91f6-4d23-91e3-ec3406df2a19"}
{"level":"info","ts":"2026-06-11T10:16:31.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"df6290a3-91f6-4d23-91e3-ec3406df2a19","RayCluster":"long-running-g6v6m","State":""}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7"}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7"}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","1 head service found":"long-running-g6v6m-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","Found 1 head Pod":"long-running-g6v6m-head-9tfk7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","head Pod":"long-running-g6v6m-head-9tfk7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-g6v6m-head-9tfk7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","worker Pod":"long-running-g6v6m-small-group-worker-z454z","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-g6v6m-small-group-worker-z454z. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:16:33.758Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"7a368516-1663-42f8-83be-8d9e446f84a7","name":"long-running-g6v6m","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:33Z","stateTransitionTimes":{"ready":"2026-06-11T10:16:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.23","serviceIP":"10.133.0.23","podName":"long-running-g6v6m-head-9tfk7","serviceName":"long-running-g6v6m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:16:29Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:16:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:08Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:08Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:33.771Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6e250a8-c306-458d-aec8-7d5bd240f9d2","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.771Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fbc8f044-08ea-44ce-98ab-26bacf46ee5d","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.771Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6e250a8-c306-458d-aec8-7d5bd240f9d2","pod":"long-running-g6v6m-head-9tfk7","ip":"10.133.0.23"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6e250a8-c306-458d-aec8-7d5bd240f9d2","pod":"long-running-g6v6m-small-group-worker-z454z","ip":"10.134.0.29"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"e6e250a8-c306-458d-aec8-7d5bd240f9d2","rayCluster":"long-running-g6v6m","podIPs":["10.133.0.23","10.134.0.29"]}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","head service name":"long-running-g6v6m-head-svc","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","head service URL":"long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682"}
{"level":"info","ts":"2026-06-11T10:16:33.772Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","command":["if","!","ray","job","status","--address","http://long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:8265","long-running-g4jns",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:8265","--no-wait","--submission-id","long-running-g4jns","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:8265","--follow","long-running-g4jns"]}
{"level":"info","ts":"2026-06-11T10:16:33.781Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-11T10:16:33.781Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","SubmissionMode":"K8sJobMode","RayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-g4jns","rayClusterName":"long-running-g6v6m","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:08Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-g4jns","rayClusterName":"long-running-g6v6m","dashboardURL":"long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:16:08Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:33.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"280f9db7-48a8-4bde-b736-474d4e75e682","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:16:33.784Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fbc8f044-08ea-44ce-98ab-26bacf46ee5d","name":"long-running-g6v6m-head"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","head service name":"long-running-g6v6m-head-svc","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","head service URL":"long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","SubmissionMode":"K8sJobMode","RayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-g4jns","rayClusterName":"long-running-g6v6m","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:08Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-g4jns","rayClusterName":"long-running-g6v6m","dashboardURL":"long-running-g6v6m-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:16:08Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:33.794Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:16:33.797Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fbc8f044-08ea-44ce-98ab-26bacf46ee5d","name":"long-running-g6v6m-workers"}
{"level":"info","ts":"2026-06-11T10:16:33.797Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"fbc8f044-08ea-44ce-98ab-26bacf46ee5d","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.803Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:16:33.803Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f"}
{"level":"error","ts":"2026-06-11T10:16:33.803Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"2b403317-f855-437a-8e47-f82e0dffb24f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.803Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6875aa00-d74f-4e33-87a1-e6f39c776186","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:33.803Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6875aa00-d74f-4e33-87a1-e6f39c776186","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.803Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6875aa00-d74f-4e33-87a1-e6f39c776186","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:33.804Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6875aa00-d74f-4e33-87a1-e6f39c776186"}
{"level":"error","ts":"2026-06-11T10:16:33.834Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6875aa00-d74f-4e33-87a1-e6f39c776186","JobId":"long-running-g4jns","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.834Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6875aa00-d74f-4e33-87a1-e6f39c776186"}
{"level":"error","ts":"2026-06-11T10:16:33.834Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"6875aa00-d74f-4e33-87a1-e6f39c776186","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.834Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9cd8744b-64ea-4157-b9c6-93f58cd4b055","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:33.834Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9cd8744b-64ea-4157-b9c6-93f58cd4b055","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.834Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9cd8744b-64ea-4157-b9c6-93f58cd4b055","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:33.834Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9cd8744b-64ea-4157-b9c6-93f58cd4b055"}
{"level":"error","ts":"2026-06-11T10:16:33.839Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9cd8744b-64ea-4157-b9c6-93f58cd4b055","JobId":"long-running-g4jns","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.839Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9cd8744b-64ea-4157-b9c6-93f58cd4b055"}
{"level":"error","ts":"2026-06-11T10:16:33.839Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"9cd8744b-64ea-4157-b9c6-93f58cd4b055","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.845Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"3000ac62-3100-441c-b506-9ac463e0b493","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:33.845Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"3000ac62-3100-441c-b506-9ac463e0b493","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.845Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"3000ac62-3100-441c-b506-9ac463e0b493","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:33.845Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"3000ac62-3100-441c-b506-9ac463e0b493"}
{"level":"error","ts":"2026-06-11T10:16:33.849Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"3000ac62-3100-441c-b506-9ac463e0b493","JobId":"long-running-g4jns","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.849Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"3000ac62-3100-441c-b506-9ac463e0b493"}
{"level":"error","ts":"2026-06-11T10:16:33.849Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"3000ac62-3100-441c-b506-9ac463e0b493","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.890Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"83f43571-df8a-4d10-b5d6-274e8b847c2e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:33.890Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"83f43571-df8a-4d10-b5d6-274e8b847c2e","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.890Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"83f43571-df8a-4d10-b5d6-274e8b847c2e","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:33.890Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"83f43571-df8a-4d10-b5d6-274e8b847c2e"}
{"level":"error","ts":"2026-06-11T10:16:33.895Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"83f43571-df8a-4d10-b5d6-274e8b847c2e","JobId":"long-running-g4jns","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.895Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"83f43571-df8a-4d10-b5d6-274e8b847c2e"}
{"level":"error","ts":"2026-06-11T10:16:33.895Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"83f43571-df8a-4d10-b5d6-274e8b847c2e","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.975Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"28584517-62bc-4bb5-9d27-f7908b964f5b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:33.975Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"28584517-62bc-4bb5-9d27-f7908b964f5b","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:33.975Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"28584517-62bc-4bb5-9d27-f7908b964f5b","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:33.975Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"28584517-62bc-4bb5-9d27-f7908b964f5b"}
{"level":"error","ts":"2026-06-11T10:16:33.980Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"28584517-62bc-4bb5-9d27-f7908b964f5b","JobId":"long-running-g4jns","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:33.980Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"28584517-62bc-4bb5-9d27-f7908b964f5b"}
{"level":"error","ts":"2026-06-11T10:16:33.980Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"28584517-62bc-4bb5-9d27-f7908b964f5b","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.141Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"925c0ea7-c352-4c6c-9551-478bb66c7512","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.141Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"925c0ea7-c352-4c6c-9551-478bb66c7512","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.141Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"925c0ea7-c352-4c6c-9551-478bb66c7512","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:34.141Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"925c0ea7-c352-4c6c-9551-478bb66c7512"}
{"level":"error","ts":"2026-06-11T10:16:34.147Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"925c0ea7-c352-4c6c-9551-478bb66c7512","JobId":"long-running-g4jns","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.147Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"925c0ea7-c352-4c6c-9551-478bb66c7512"}
{"level":"error","ts":"2026-06-11T10:16:34.147Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"925c0ea7-c352-4c6c-9551-478bb66c7512","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.467Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"bebbcbea-0e8c-41e9-9ce1-c52df3902753","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.467Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"bebbcbea-0e8c-41e9-9ce1-c52df3902753","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.467Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"bebbcbea-0e8c-41e9-9ce1-c52df3902753","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:34.467Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"bebbcbea-0e8c-41e9-9ce1-c52df3902753"}
{"level":"error","ts":"2026-06-11T10:16:34.473Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"bebbcbea-0e8c-41e9-9ce1-c52df3902753","JobId":"long-running-g4jns","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.473Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"bebbcbea-0e8c-41e9-9ce1-c52df3902753"}
{"level":"error","ts":"2026-06-11T10:16:34.473Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"bebbcbea-0e8c-41e9-9ce1-c52df3902753","error":"Job long-running-g4jns does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.749Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"98d2d5bb-3989-49a3-a42d-f15dc9aff67b","DeletionTimestamp":"2026-06-11 10:16:34 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:16:34.749Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"98d2d5bb-3989-49a3-a42d-f15dc9aff67b","rayJob":"long-running-g4jns"}
{"level":"error","ts":"2026-06-11T10:16:34.755Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"98d2d5bb-3989-49a3-a42d-f15dc9aff67b","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.755Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"98d2d5bb-3989-49a3-a42d-f15dc9aff67b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:16:34.766Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71b040d4-ba42-4af1-8c30-b9a128e09758","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.766Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71b040d4-ba42-4af1-8c30-b9a128e09758","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:16:34.779Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71b040d4-ba42-4af1-8c30-b9a128e09758"}
{"level":"info","ts":"2026-06-11T10:16:34.779Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71b040d4-ba42-4af1-8c30-b9a128e09758","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:16:34.779Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71b040d4-ba42-4af1-8c30-b9a128e09758","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-59c8g","rayClusterName":"counter-8qrct","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:34.779Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71b040d4-ba42-4af1-8c30-b9a128e09758","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"b82144e3-26d8-4ce8-917b-e73c4422e551","rayCluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"reconcileID":"e68cf858-43e5-41bd-b9e1-207a1d77522b"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","namespace":"test-ns-v8w6c","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"900fd20f-aee5-4449-8e01-1321350c0a23"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"b82144e3-26d8-4ce8-917b-e73c4422e551","pod":"long-running-g6v6m-head-9tfk7","ip":"10.133.0.23"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"b82144e3-26d8-4ce8-917b-e73c4422e551","pod":"long-running-g6v6m-small-group-worker-z454z","ip":"10.134.0.29"}
{"level":"info","ts":"2026-06-11T10:16:34.790Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"b82144e3-26d8-4ce8-917b-e73c4422e551","rayCluster":"long-running-g6v6m","podIPs":["10.133.0.23","10.134.0.29"]}
{"level":"info","ts":"2026-06-11T10:16:34.792Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f0461788-44b1-4219-bf7d-262c23038b57"}
{"level":"info","ts":"2026-06-11T10:16:34.792Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.792Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796"}
{"level":"info","ts":"2026-06-11T10:16:34.792Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:16:34.792Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-srwmp","rayClusterName":"counter-ft5ch","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:34.792Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:16:34.800Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","name":"kuberay-gateway-access","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:34.801Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:16:34.801Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796"}
{"level":"error","ts":"2026-06-11T10:16:34.801Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c821c763-bec7-4506-83c3-f1ebf82be796","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.801Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ef6ce0a6-59bd-4d22-8277-b8bf28c20076","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.801Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ef6ce0a6-59bd-4d22-8277-b8bf28c20076","oldStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:16:34.801Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ef6ce0a6-59bd-4d22-8277-b8bf28c20076","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-59c8g","rayClusterName":"counter-8qrct","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-59c8g","rayClusterName":"counter-8qrct","jobDeploymentStatus":"Suspending","startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:34.801Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ef6ce0a6-59bd-4d22-8277-b8bf28c20076","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-06-11T10:16:34.827Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","name":"test-ns-v8w6c-long-running-g6v6m","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:16:34.828Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3a4e3fba-e143-4d96-8107-d7a5fd09e725","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.828Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3a4e3fba-e143-4d96-8107-d7a5fd09e725","RayCluster":{"name":"counter-8qrct","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:34.828Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3a4e3fba-e143-4d96-8107-d7a5fd09e725","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:16:34.828Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3a4e3fba-e143-4d96-8107-d7a5fd09e725","Kubernetes Job":""}
{"level":"info","ts":"2026-06-11T10:16:34.828Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3a4e3fba-e143-4d96-8107-d7a5fd09e725","isJobDeleted":true}
{"level":"info","ts":"2026-06-11T10:16:34.828Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3a4e3fba-e143-4d96-8107-d7a5fd09e725","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-59c8g","rayClusterName":"counter-8qrct","jobDeploymentStatus":"Suspending","startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:34.828Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3a4e3fba-e143-4d96-8107-d7a5fd09e725","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-06-11T10:16:34.833Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","configMap":"kube-rbac-proxy-config-long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.839Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1c796d98-0837-4ced-9b7e-2175fa186dfe","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.840Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"96186779-b68c-41e4-9ba9-b9568603aa9d","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:34.840Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","serviceAccount":"long-running-g6v6m-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:34.849Z","logger":"raycluster-resource","msg":"default","name":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.849Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:34.860Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"7c07cdba-dfc2-4f25-80fa-e22d4301803f","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.860Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:34.860Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.860Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe","cluster":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.860Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe","namespace":"test-ns-v8w6c","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:16:34.860Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:16:34.861Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"afc9e545-6021-465b-a675-0f47f990d8b9","namespace":"test-ns-v8w6c","clusterName":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.861Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"762852ad-f4ae-4e40-af00-203193c60fd9"}
{"level":"info","ts":"2026-06-11T10:16:34.861Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"f67109de-7152-4610-becc-0bdaa1459390"}
{"level":"error","ts":"2026-06-11T10:16:34.864Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe","error":"rayclusters.ray.io \"long-running-g6v6m\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.927Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe"}
{"level":"error","ts":"2026-06-11T10:16:34.927Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"5f1f3d3d-769d-441b-bc30-a410e75568fe","error":"rayclusters.ray.io \"long-running-g6v6m\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.927Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"67201076-e3e4-4f39-9154-7d1262c34f54","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:34.927Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"67201076-e3e4-4f39-9154-7d1262c34f54"}
{"level":"info","ts":"2026-06-11T10:16:34.927Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"67201076-e3e4-4f39-9154-7d1262c34f54","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:16:34.927Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"67201076-e3e4-4f39-9154-7d1262c34f54","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:34.927Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"1551c5e0-06c6-4b10-859a-3b05ea9d25d7"}
{"level":"info","ts":"2026-06-11T10:16:34.927Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"b7e347bd-f14a-41c5-91db-4f93b4d0197a"}
{"level":"error","ts":"2026-06-11T10:16:34.931Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"afc9e545-6021-465b-a675-0f47f990d8b9","certificate":"ray-head-cert-long-running-g6v6m","error":"certificates.cert-manager.io \"ray-head-cert-long-running-g6v6m\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.932Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"90e09810-1cfa-439c-b042-5878d85aaa7c","namespacedName":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:34.932Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"90e09810-1cfa-439c-b042-5878d85aaa7c"}
{"level":"info","ts":"2026-06-11T10:16:34.932Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"90e09810-1cfa-439c-b042-5878d85aaa7c","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:16:34.932Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"90e09810-1cfa-439c-b042-5878d85aaa7c","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:34.949Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"afc9e545-6021-465b-a675-0f47f990d8b9","errorCount":1}
{"level":"info","ts":"2026-06-11T10:16:34.949Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"afc9e545-6021-465b-a675-0f47f990d8b9"}
{"level":"error","ts":"2026-06-11T10:16:34.949Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"afc9e545-6021-465b-a675-0f47f990d8b9","error":"certificates.cert-manager.io \"ray-head-cert-long-running-g6v6m\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:34.954Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"3086d73a-9e7e-4d9f-a9fa-9f393b967b33","namespace":"test-ns-v8w6c","clusterName":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:34.954Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-g6v6m","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"long-running-g6v6m","reconcileID":"3086d73a-9e7e-4d9f-a9fa-9f393b967b33","namespace":"test-ns-v8w6c","clusterName":"long-running-g6v6m"}
{"level":"info","ts":"2026-06-11T10:16:35.127Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-v8w6c"},"reconcileID":"d0ad59fb-7d7c-46c1-aa39-c79b4f81ade3"}
{"level":"info","ts":"2026-06-11T10:16:35.785Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"799ddf08-f7b9-4990-bee2-4706697f55d2","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:35.785Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"799ddf08-f7b9-4990-bee2-4706697f55d2"}
{"level":"info","ts":"2026-06-11T10:16:35.785Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"799ddf08-f7b9-4990-bee2-4706697f55d2","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:35.785Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"799ddf08-f7b9-4990-bee2-4706697f55d2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-06-11T10:16:35.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d56e414f-ab0c-4231-9121-23cab136d424","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:35.797Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d56e414f-ab0c-4231-9121-23cab136d424"}
{"level":"info","ts":"2026-06-11T10:16:35.797Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d56e414f-ab0c-4231-9121-23cab136d424","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:16:35.797Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d56e414f-ab0c-4231-9121-23cab136d424","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-11T10:16:34Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:16:35.797Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d56e414f-ab0c-4231-9121-23cab136d424","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:16:35.809Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3cce4abd-7f1b-45a5-8cd7-bc083477d912","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:35.809Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3cce4abd-7f1b-45a5-8cd7-bc083477d912","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.809Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3cce4abd-7f1b-45a5-8cd7-bc083477d912","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.809Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3cce4abd-7f1b-45a5-8cd7-bc083477d912","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.814Z","logger":"raycluster-resource","msg":"default","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.814Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-dqqkt","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:35.830Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3cce4abd-7f1b-45a5-8cd7-bc083477d912","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3cce4abd-7f1b-45a5-8cd7-bc083477d912"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3cce4abd-7f1b-45a5-8cd7-bc083477d912","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b59ba541-890f-46c7-baca-759437b42fbf","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","mode":"IntegratedOAuth","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5887fae5-24f1-4dff-8df4-1b27a98a21b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5887fae5-24f1-4dff-8df4-1b27a98a21b1","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5887fae5-24f1-4dff-8df4-1b27a98a21b1","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5887fae5-24f1-4dff-8df4-1b27a98a21b1"}
{"level":"info","ts":"2026-06-11T10:16:35.831Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5887fae5-24f1-4dff-8df4-1b27a98a21b1","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:35.837Z","logger":"raycluster-resource","msg":"default","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.837Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-dqqkt","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:16:35.840Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","name":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:35.840Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:35.840Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.840Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:35.840Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.840Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","serviceSelectors":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/identifier":"counter-dqqkt-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.840Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","error":"unable to find head service. cluster name counter-dqqkt, filter labels [test-ns-v8w6c map[ray.io/cluster:counter-dqqkt ray.io/identifier:counter-dqqkt-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32"}
{"level":"error","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb5356-d71f-4dcd-8eb9-f9a939b5ed32","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874"}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874"}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:16:35.841Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","name":"counter-dqqkt","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:35.842Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.842Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.842Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b59ba541-890f-46c7-baca-759437b42fbf","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.842Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"e87fadf4-a6ca-443f-84b6-d994e5219bee","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.846Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2386e0a1-fc85-46cd-a2d8-f7096ca8a6f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:35.846Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2386e0a1-fc85-46cd-a2d8-f7096ca8a6f7","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.846Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2386e0a1-fc85-46cd-a2d8-f7096ca8a6f7","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2386e0a1-fc85-46cd-a2d8-f7096ca8a6f7"}
{"level":"info","ts":"2026-06-11T10:16:35.847Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2386e0a1-fc85-46cd-a2d8-f7096ca8a6f7","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:35.852Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","name":"counter-dqqkt-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:35.853Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","name":"counter-dqqkt","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-dqqkt\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-v8w6c","name":"counter-dqqkt"}}
{"level":"info","ts":"2026-06-11T10:16:35.853Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874"}
{"level":"error","ts":"2026-06-11T10:16:35.853Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"178eb4d2-d7e5-436e-9dcc-87408c6d7874","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71"}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71"}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:16:35.854Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","name":"counter-dqqkt","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:35.927Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771"}
{"level":"info","ts":"2026-06-11T10:16:35.927Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"e87fadf4-a6ca-443f-84b6-d994e5219bee","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:16:35.927Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","name":"kuberay-gateway-access","namespace":"test-ns-v8w6c","operation":"created"}
{"level":"info","ts":"2026-06-11T10:16:35.927Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:16:35.927Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.937Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","name":"test-ns-v8w6c-counter-dqqkt","operation":"created"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5808ad56-7105-4552-b527-34ea78a754a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5808ad56-7105-4552-b527-34ea78a754a8","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5808ad56-7105-4552-b527-34ea78a754a8","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5808ad56-7105-4552-b527-34ea78a754a8"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5808ad56-7105-4552-b527-34ea78a754a8","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71"}
{"level":"error","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ccd17632-063c-40c0-abb0-f4cf11c85d71","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf"}
{"level":"error","ts":"2026-06-11T10:16:35.942Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"e5aba4aa-5f4c-4d94-aa83-dc15b28deccf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.947Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"e87fadf4-a6ca-443f-84b6-d994e5219bee","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:16:35.947Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","name":"kube-rbac-proxy-config-counter-dqqkt","operation":"created"}
{"level":"info","ts":"2026-06-11T10:16:35.947Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"e87fadf4-a6ca-443f-84b6-d994e5219bee","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.947Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt"}
{"level":"error","ts":"2026-06-11T10:16:35.954Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-dqqkt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.954Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae"}
{"level":"error","ts":"2026-06-11T10:16:35.954Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a7403504-c2c4-4e75-8c24-84c89c9da0ae","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-dqqkt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:35.954Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"22c77388-f233-4a59-b56b-4c2c7c262333","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:35.954Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"22c77388-f233-4a59-b56b-4c2c7c262333","mode":"IntegratedOAuth","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:35.954Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"22c77388-f233-4a59-b56b-4c2c7c262333","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.027Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b5bdf69c-aa18-4fde-8046-4185da25fc5a","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196"}
{"level":"info","ts":"2026-06-11T10:16:36.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196"}
{"level":"info","ts":"2026-06-11T10:16:36.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:36.027Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:36.028Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196"}
{"level":"error","ts":"2026-06-11T10:16:36.028Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c4ee8a2-03e9-4b69-b532-a10342fc1196","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771"}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"22c77388-f233-4a59-b56b-4c2c7c262333","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7577b471-523e-4a8d-aa5f-5dd480c53554","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7577b471-523e-4a8d-aa5f-5dd480c53554","mode":"IntegratedOAuth","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7577b471-523e-4a8d-aa5f-5dd480c53554","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.028Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:16:36.039Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7577b471-523e-4a8d-aa5f-5dd480c53554","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.040Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771"}
{"level":"info","ts":"2026-06-11T10:16:36.040Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","rayCluster":"counter-dqqkt","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:16:36.040Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-dqqkt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.040Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771"}
{"level":"error","ts":"2026-06-11T10:16:36.040Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"996d2abf-074c-4354-a13a-412bbb810771","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-dqqkt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.040Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7dd0626c-6a9f-4fce-b1c5-50dadf40c4f1","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.043Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b5bdf69c-aa18-4fde-8046-4185da25fc5a","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7dd0626c-6a9f-4fce-b1c5-50dadf40c4f1"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:36.127Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7dd0626c-6a9f-4fce-b1c5-50dadf40c4f1","error":"certificates.cert-manager.io \"ray-worker-cert-counter-dqqkt\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:16:36.127Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7dd0626c-6a9f-4fce-b1c5-50dadf40c4f1","error":"certificates.cert-manager.io \"ray-worker-cert-counter-dqqkt\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5cbf686d-d612-4716-ab88-d3a8d670eb13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7dd0626c-6a9f-4fce-b1c5-50dadf40c4f1"}
{"level":"error","ts":"2026-06-11T10:16:36.127Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7dd0626c-6a9f-4fce-b1c5-50dadf40c4f1","error":"certificates.cert-manager.io \"ray-worker-cert-counter-dqqkt\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.128Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.128Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7fd848a5-c963-4772-96eb-7571ac728322","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.128Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e"}
{"level":"error","ts":"2026-06-11T10:16:36.128Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"72a7404a-6f57-4219-a514-5ca05fd6487e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.127Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5cbf686d-d612-4716-ab88-d3a8d670eb13","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.128Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5cbf686d-d612-4716-ab88-d3a8d670eb13","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:36.128Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5cbf686d-d612-4716-ab88-d3a8d670eb13"}
{"level":"info","ts":"2026-06-11T10:16:36.128Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"5cbf686d-d612-4716-ab88-d3a8d670eb13","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:36.227Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b5bdf69c-aa18-4fde-8046-4185da25fc5a","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:16:36.227Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b5bdf69c-aa18-4fde-8046-4185da25fc5a","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.227Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7fd848a5-c963-4772-96eb-7571ac728322"}
{"level":"info","ts":"2026-06-11T10:16:36.227Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"afb8a4d8-f1b8-4381-904d-787bd7e656af","cluster":"counter-dqqkt"}
{"level":"error","ts":"2026-06-11T10:16:36.227Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7fd848a5-c963-4772-96eb-7571ac728322","error":"certificates.cert-manager.io \"ray-worker-cert-counter-dqqkt\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:16:36.227Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7fd848a5-c963-4772-96eb-7571ac728322","error":"certificates.cert-manager.io \"ray-worker-cert-counter-dqqkt\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.227Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7fd848a5-c963-4772-96eb-7571ac728322"}
{"level":"error","ts":"2026-06-11T10:16:36.227Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"7fd848a5-c963-4772-96eb-7571ac728322","error":"certificates.cert-manager.io \"ray-worker-cert-counter-dqqkt\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.227Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"9222f93c-faf5-4800-b2f2-5266ef252438","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.228Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"9222f93c-faf5-4800-b2f2-5266ef252438"}
{"level":"info","ts":"2026-06-11T10:16:36.241Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"afb8a4d8-f1b8-4381-904d-787bd7e656af","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:16:36.248Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"9b302ca5-cf31-4ad2-9115-bc9cdba53bb7","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.248Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"9b302ca5-cf31-4ad2-9115-bc9cdba53bb7"}
{"level":"info","ts":"2026-06-11T10:16:36.255Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"afb8a4d8-f1b8-4381-904d-787bd7e656af","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:16:36.255Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"afb8a4d8-f1b8-4381-904d-787bd7e656af","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e"}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e"}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e"}
{"level":"error","ts":"2026-06-11T10:16:36.288Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"ae74f2a1-70e2-40f1-b7ee-ec808d863f8e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53"}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53"}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53"}
{"level":"error","ts":"2026-06-11T10:16:36.609Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"58e4b44f-e37e-4280-b4b9-a1a01b66cb53","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4"}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4"}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","error":"head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"error","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found"}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","filter labels":["test-ns-v8w6c",{"ray.io/cluster":"counter-dqqkt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4"}
{"level":"error","ts":"2026-06-11T10:16:37.250Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"067ea87b-9774-4d44-b7f6-ccde794804f4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-dqqkt not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:37.829Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"38814d53-c610-40c3-a6d7-e9f97de62a8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:37.829Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"38814d53-c610-40c3-a6d7-e9f97de62a8f","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:37.829Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"38814d53-c610-40c3-a6d7-e9f97de62a8f","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:37.829Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"38814d53-c610-40c3-a6d7-e9f97de62a8f"}
{"level":"info","ts":"2026-06-11T10:16:37.829Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"38814d53-c610-40c3-a6d7-e9f97de62a8f","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","mode":"IntegratedOAuth","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","cluster":"counter-dqqkt","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","cluster":"counter-dqqkt","authType":"IntegratedOAuth","serviceAccount":"counter-dqqkt-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-dqqkt","ray.io/group":"headgroup","ray.io/identifier":"counter-dqqkt-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:16:38.530Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","enabled":true}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","name":"counter-dqqkt-head-rjvd6"}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3"}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","nodeType":"worker","rayStartParams":{"address":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","rayStartCmd":"ray start  --address=counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:16:38.548Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","enabled":true}
{"level":"info","ts":"2026-06-11T10:16:38.562Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","name":"counter-dqqkt-small-group-worker-shz5x"}
{"level":"info","ts":"2026-06-11T10:16:38.562Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.562Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.562Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-dqqkt-head-svc"},"newHead":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"}}
{"level":"info","ts":"2026-06-11T10:16:38.562Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"7d3d406d-45a3-44a3-9863-c6523f7a89f3","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:38.577Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f"}
{"level":"info","ts":"2026-06-11T10:16:38.577Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f"}
{"level":"info","ts":"2026-06-11T10:16:38.577Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-dqqkt-head-svc"},"newHead":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"}}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"032e5be0-7b0f-428e-aacc-fcf5dc60ac32","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"da494cc4-7ec6-4c41-a6c2-88bcbb93529b","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"da494cc4-7ec6-4c41-a6c2-88bcbb93529b","rayCluster":"counter-dqqkt","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ed61e131-827e-43be-907a-ba32ece93793","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ed61e131-827e-43be-907a-ba32ece93793","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ed61e131-827e-43be-907a-ba32ece93793","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:38.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ed61e131-827e-43be-907a-ba32ece93793"}
{"level":"info","ts":"2026-06-11T10:16:38.579Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ed61e131-827e-43be-907a-ba32ece93793","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","name":"counter-dqqkt","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-dqqkt\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-v8w6c","name":"counter-dqqkt"}}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f"}
{"level":"error","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1e835731-74f6-4e33-8083-e41637d32c6f","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-dqqkt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","podName":"counter-dqqkt-small-group-worker-shz5x","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","podName":"counter-dqqkt-small-group-worker-shz5x","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:38.585Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"89500287-56ac-4377-9d33-62d37afa1d2e","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"032e5be0-7b0f-428e-aacc-fcf5dc60ac32","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.627Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.628Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:16:38.628Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"13a3cf8c-ea63-466c-a322-a0813b9ca011","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:38.639Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"032e5be0-7b0f-428e-aacc-fcf5dc60ac32","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:16:38.639Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"032e5be0-7b0f-428e-aacc-fcf5dc60ac32","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0ccd9df2-747e-49a8-9dec-e44e8e646620","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dd36e5e2-2eb7-41f4-85a1-f8bd523ad207","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0ccd9df2-747e-49a8-9dec-e44e8e646620","rayCluster":"counter-dqqkt","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9737870c-6310-4c19-9a40-a2e0c79a1dbc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9737870c-6310-4c19-9a40-a2e0c79a1dbc","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.640Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9737870c-6310-4c19-9a40-a2e0c79a1dbc","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:38.641Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9737870c-6310-4c19-9a40-a2e0c79a1dbc"}
{"level":"info","ts":"2026-06-11T10:16:38.641Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9737870c-6310-4c19-9a40-a2e0c79a1dbc","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:38.647Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","name":"counter-dqqkt","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-dqqkt\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-v8w6c","name":"counter-dqqkt"}}
{"level":"info","ts":"2026-06-11T10:16:38.647Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f"}
{"level":"error","ts":"2026-06-11T10:16:38.647Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9e856541-e8b8-4ad6-88f3-86d20489ba6f","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-dqqkt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dd36e5e2-2eb7-41f4-85a1-f8bd523ad207","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:38.653Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"15ceb736-9217-4330-848f-82ece0a28fe3","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:38.665Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dd36e5e2-2eb7-41f4-85a1-f8bd523ad207","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:16:38.665Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dd36e5e2-2eb7-41f4-85a1-f8bd523ad207","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:38.999Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"6be76bee-630c-4552-98e8-87c08a60fbd6","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:39.018Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c"}
{"level":"info","ts":"2026-06-11T10:16:39.018Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c"}
{"level":"info","ts":"2026-06-11T10:16:39.018Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:39.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:39.019Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"2c8c2d98-e358-4f10-9010-096cc67ff81c","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"newHead":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"}}
{"level":"info","ts":"2026-06-11T10:16:39.533Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"587464b4-7574-4a48-afb0-ccdfca13da55","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0a8ae4da-72ab-4eac-8a71-173f2a8e9dd3","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c6c1267e-0911-445e-9d69-79f6bc689f82","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0a8ae4da-72ab-4eac-8a71-173f2a8e9dd3","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e6bc68ad-382b-461a-8865-fa8e075290a4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e6bc68ad-382b-461a-8865-fa8e075290a4","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e6bc68ad-382b-461a-8865-fa8e075290a4","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e6bc68ad-382b-461a-8865-fa8e075290a4"}
{"level":"info","ts":"2026-06-11T10:16:39.546Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e6bc68ad-382b-461a-8865-fa8e075290a4","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:39.558Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c6c1267e-0911-445e-9d69-79f6bc689f82","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:16:39.558Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0a8ae4da-72ab-4eac-8a71-173f2a8e9dd3","certificate":"ray-head-cert-counter-dqqkt","dnsNames":["counter-dqqkt-head-svc","localhost","counter-dqqkt-head-svc.test-ns-v8w6c.svc","counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:16:39.571Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c6c1267e-0911-445e-9d69-79f6bc689f82","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:16:39.571Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c6c1267e-0911-445e-9d69-79f6bc689f82","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:39.572Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0a8ae4da-72ab-4eac-8a71-173f2a8e9dd3","certificate":"ray-worker-cert-counter-dqqkt","dnsNames":["counter-dqqkt-worker-svc","localhost","counter-dqqkt-small-group","counter-dqqkt-worker-svc.test-ns-v8w6c.svc","counter-dqqkt-worker-svc.test-ns-v8w6c.svc.cluster.local","*.counter-dqqkt-worker-svc.test-ns-v8w6c.svc","*.counter-dqqkt-worker-svc.test-ns-v8w6c.svc.cluster.local","*-worker-*.test-ns-v8w6c.svc","*-worker-*.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:16:39.572Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0a8ae4da-72ab-4eac-8a71-173f2a8e9dd3","rayCluster":"counter-dqqkt","podIPs":["10.133.0.24"]}
{"level":"info","ts":"2026-06-11T10:16:39.811Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8"}
{"level":"info","ts":"2026-06-11T10:16:39.811Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8"}
{"level":"info","ts":"2026-06-11T10:16:39.811Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:39.811Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:39.812Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"3e43bf51-73a2-4f18-8b25-181c2f28c2d8","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:40.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"706c971c-366b-4f2b-93ee-af0c72d353c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:40.830Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"706c971c-366b-4f2b-93ee-af0c72d353c6","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:40.830Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"706c971c-366b-4f2b-93ee-af0c72d353c6","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:40.830Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"706c971c-366b-4f2b-93ee-af0c72d353c6"}
{"level":"info","ts":"2026-06-11T10:16:40.830Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"706c971c-366b-4f2b-93ee-af0c72d353c6","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:16:41.546Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"eae9ae52-4624-4d3a-8b83-f6e6497e610f","seconds":300}
{"level":"info","ts":"2026-06-11T10:16:43.831Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8c4064da-1c72-4e55-8d59-938e5cdab9f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:43.831Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8c4064da-1c72-4e55-8d59-938e5cdab9f9","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:43.831Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8c4064da-1c72-4e55-8d59-938e5cdab9f9","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:43.831Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8c4064da-1c72-4e55-8d59-938e5cdab9f9"}
{"level":"info","ts":"2026-06-11T10:16:43.831Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8c4064da-1c72-4e55-8d59-938e5cdab9f9","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:46.832Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"389150b2-d1bb-40bd-b19b-efe056d1f0cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:46.832Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"389150b2-d1bb-40bd-b19b-efe056d1f0cb","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:46.832Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"389150b2-d1bb-40bd-b19b-efe056d1f0cb","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:46.832Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"389150b2-d1bb-40bd-b19b-efe056d1f0cb"}
{"level":"info","ts":"2026-06-11T10:16:46.832Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"389150b2-d1bb-40bd-b19b-efe056d1f0cb","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:49.833Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"fb3a2b7b-7e9a-40ad-816c-67d12ff9f2c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:49.833Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"fb3a2b7b-7e9a-40ad-816c-67d12ff9f2c6","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:49.833Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"fb3a2b7b-7e9a-40ad-816c-67d12ff9f2c6","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:49.833Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"fb3a2b7b-7e9a-40ad-816c-67d12ff9f2c6"}
{"level":"info","ts":"2026-06-11T10:16:49.833Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"fb3a2b7b-7e9a-40ad-816c-67d12ff9f2c6","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:52.834Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e7b374d2-de1f-463a-a7e8-f1c79530a118","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:52.834Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e7b374d2-de1f-463a-a7e8-f1c79530a118","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:52.834Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e7b374d2-de1f-463a-a7e8-f1c79530a118","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:52.835Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e7b374d2-de1f-463a-a7e8-f1c79530a118"}
{"level":"info","ts":"2026-06-11T10:16:52.835Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e7b374d2-de1f-463a-a7e8-f1c79530a118","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:55.835Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71adae39-c042-4ca7-9b7e-9d60701f349e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:55.835Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71adae39-c042-4ca7-9b7e-9d60701f349e","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:55.835Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71adae39-c042-4ca7-9b7e-9d60701f349e","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:55.835Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71adae39-c042-4ca7-9b7e-9d60701f349e"}
{"level":"info","ts":"2026-06-11T10:16:55.835Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"71adae39-c042-4ca7-9b7e-9d60701f349e","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:16:58.836Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"988aecf7-9b1f-4282-8751-5ac1904c0744","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:16:58.836Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"988aecf7-9b1f-4282-8751-5ac1904c0744","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:16:58.836Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"988aecf7-9b1f-4282-8751-5ac1904c0744","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:16:58.836Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"988aecf7-9b1f-4282-8751-5ac1904c0744"}
{"level":"info","ts":"2026-06-11T10:16:58.836Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"988aecf7-9b1f-4282-8751-5ac1904c0744","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:01.837Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c731c431-413e-4ec1-a228-041803ecb093","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:01.837Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c731c431-413e-4ec1-a228-041803ecb093","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:01.837Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c731c431-413e-4ec1-a228-041803ecb093","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:01.837Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c731c431-413e-4ec1-a228-041803ecb093"}
{"level":"info","ts":"2026-06-11T10:17:01.837Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c731c431-413e-4ec1-a228-041803ecb093","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:04.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"50710310-a0eb-4cd4-b4d9-2a06a2034cf7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:04.838Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"50710310-a0eb-4cd4-b4d9-2a06a2034cf7","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:04.838Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"50710310-a0eb-4cd4-b4d9-2a06a2034cf7","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:04.838Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"50710310-a0eb-4cd4-b4d9-2a06a2034cf7"}
{"level":"info","ts":"2026-06-11T10:17:04.838Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"50710310-a0eb-4cd4-b4d9-2a06a2034cf7","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:06.249Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a9d3a3de-6876-4a65-84fe-646bddb5b425","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:06.249Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a9d3a3de-6876-4a65-84fe-646bddb5b425","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:17:06.249Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a9d3a3de-6876-4a65-84fe-646bddb5b425","pod":"counter-dqqkt-small-group-worker-shz5x","ip":"10.134.0.31"}
{"level":"info","ts":"2026-06-11T10:17:06.267Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a9d3a3de-6876-4a65-84fe-646bddb5b425","certificate":"ray-head-cert-counter-dqqkt","dnsNames":["counter-dqqkt-head-svc","localhost","counter-dqqkt-head-svc.test-ns-v8w6c.svc","counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.24","10.134.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:17:06.285Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a9d3a3de-6876-4a65-84fe-646bddb5b425","certificate":"ray-worker-cert-counter-dqqkt","dnsNames":["counter-dqqkt-worker-svc","localhost","counter-dqqkt-small-group","counter-dqqkt-worker-svc.test-ns-v8w6c.svc","counter-dqqkt-worker-svc.test-ns-v8w6c.svc.cluster.local","*.counter-dqqkt-worker-svc.test-ns-v8w6c.svc","*.counter-dqqkt-worker-svc.test-ns-v8w6c.svc.cluster.local","*-worker-*.test-ns-v8w6c.svc","*-worker-*.test-ns-v8w6c.svc.cluster.local"],"ipAddresses":["10.133.0.24","10.134.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:17:06.285Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a9d3a3de-6876-4a65-84fe-646bddb5b425","rayCluster":"counter-dqqkt","podIPs":["10.133.0.24","10.134.0.31"]}
{"level":"info","ts":"2026-06-11T10:17:07.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e623eb95-03a6-4799-85d9-87d4dc0c2047","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:07.838Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e623eb95-03a6-4799-85d9-87d4dc0c2047","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:07.838Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e623eb95-03a6-4799-85d9-87d4dc0c2047","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:07.838Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e623eb95-03a6-4799-85d9-87d4dc0c2047"}
{"level":"info","ts":"2026-06-11T10:17:07.838Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e623eb95-03a6-4799-85d9-87d4dc0c2047","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:10.839Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bd07685d-6908-4ba1-ad63-b78d7bb47420","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:10.839Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bd07685d-6908-4ba1-ad63-b78d7bb47420","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:10.839Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bd07685d-6908-4ba1-ad63-b78d7bb47420","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:10.839Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bd07685d-6908-4ba1-ad63-b78d7bb47420"}
{"level":"info","ts":"2026-06-11T10:17:10.839Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bd07685d-6908-4ba1-ad63-b78d7bb47420","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:13.839Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"450ab3e3-a502-455a-988e-7bdf31d8f0b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:13.839Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"450ab3e3-a502-455a-988e-7bdf31d8f0b4","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:13.839Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"450ab3e3-a502-455a-988e-7bdf31d8f0b4","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:13.839Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"450ab3e3-a502-455a-988e-7bdf31d8f0b4"}
{"level":"info","ts":"2026-06-11T10:17:13.839Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"450ab3e3-a502-455a-988e-7bdf31d8f0b4","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:16.584Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"51c340e8-e6bd-4d4e-babc-789c6fadcc7f","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:17:16.584Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"51c340e8-e6bd-4d4e-babc-789c6fadcc7f","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:17:16.584Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"51c340e8-e6bd-4d4e-babc-789c6fadcc7f","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:17:16.584Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"51c340e8-e6bd-4d4e-babc-789c6fadcc7f","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:17:16.840Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ac853256-9c85-41c0-a688-fb3eedc9e246","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:16.840Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ac853256-9c85-41c0-a688-fb3eedc9e246","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:16.840Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ac853256-9c85-41c0-a688-fb3eedc9e246","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:16.841Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ac853256-9c85-41c0-a688-fb3eedc9e246"}
{"level":"info","ts":"2026-06-11T10:17:16.841Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ac853256-9c85-41c0-a688-fb3eedc9e246","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:17.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5"}
{"level":"info","ts":"2026-06-11T10:17:17.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5"}
{"level":"info","ts":"2026-06-11T10:17:17.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:17:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:17:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:17:17.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:17:17.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:17:17.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:17:17.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:17:17.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:17:17.622Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:17:17.622Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"3e2ca8df-99ca-4b09-b407-9e84b1fac9e5","seconds":300}
{"level":"info","ts":"2026-06-11T10:17:19.841Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a7eb8424-0c89-4ec9-84a9-b7b625a9b25f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:19.841Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a7eb8424-0c89-4ec9-84a9-b7b625a9b25f","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:19.841Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a7eb8424-0c89-4ec9-84a9-b7b625a9b25f","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:19.842Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a7eb8424-0c89-4ec9-84a9-b7b625a9b25f"}
{"level":"info","ts":"2026-06-11T10:17:19.842Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a7eb8424-0c89-4ec9-84a9-b7b625a9b25f","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:22.842Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4579841e-e448-4931-8226-f868fa4b4fe8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:22.842Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4579841e-e448-4931-8226-f868fa4b4fe8","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:22.842Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4579841e-e448-4931-8226-f868fa4b4fe8","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:22.842Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4579841e-e448-4931-8226-f868fa4b4fe8"}
{"level":"info","ts":"2026-06-11T10:17:22.842Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4579841e-e448-4931-8226-f868fa4b4fe8","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:25.843Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"cf9cf3e9-97da-4adf-88f4-3d4d2c082a68","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:25.843Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"cf9cf3e9-97da-4adf-88f4-3d4d2c082a68","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:25.843Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"cf9cf3e9-97da-4adf-88f4-3d4d2c082a68","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:25.843Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"cf9cf3e9-97da-4adf-88f4-3d4d2c082a68"}
{"level":"info","ts":"2026-06-11T10:17:25.843Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"cf9cf3e9-97da-4adf-88f4-3d4d2c082a68","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:28.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"47950240-10d4-429b-a10b-25f18c4df355","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:28.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"47950240-10d4-429b-a10b-25f18c4df355","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:28.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"47950240-10d4-429b-a10b-25f18c4df355","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:28.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"47950240-10d4-429b-a10b-25f18c4df355"}
{"level":"info","ts":"2026-06-11T10:17:28.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"47950240-10d4-429b-a10b-25f18c4df355","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:31.845Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"66e1a60c-c1b4-431b-b3d6-ef2a18955b91","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:31.845Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"66e1a60c-c1b4-431b-b3d6-ef2a18955b91","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:31.845Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"66e1a60c-c1b4-431b-b3d6-ef2a18955b91","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:31.845Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"66e1a60c-c1b4-431b-b3d6-ef2a18955b91"}
{"level":"info","ts":"2026-06-11T10:17:31.845Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"66e1a60c-c1b4-431b-b3d6-ef2a18955b91","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:34.846Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8cd69ff9-b538-4104-a283-99548aa4d697","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:34.846Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8cd69ff9-b538-4104-a283-99548aa4d697","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:34.846Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8cd69ff9-b538-4104-a283-99548aa4d697","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:34.846Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8cd69ff9-b538-4104-a283-99548aa4d697"}
{"level":"info","ts":"2026-06-11T10:17:34.846Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"8cd69ff9-b538-4104-a283-99548aa4d697","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:37.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9966f2a8-7afc-4962-adc4-63761af06ec2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:37.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9966f2a8-7afc-4962-adc4-63761af06ec2","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:37.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9966f2a8-7afc-4962-adc4-63761af06ec2","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:37.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9966f2a8-7afc-4962-adc4-63761af06ec2"}
{"level":"info","ts":"2026-06-11T10:17:37.847Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"9966f2a8-7afc-4962-adc4-63761af06ec2","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:40.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e3bb352c-c14a-49d2-a3b6-8c77b0f9fb3d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:40.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e3bb352c-c14a-49d2-a3b6-8c77b0f9fb3d","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:40.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e3bb352c-c14a-49d2-a3b6-8c77b0f9fb3d","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:40.849Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e3bb352c-c14a-49d2-a3b6-8c77b0f9fb3d"}
{"level":"info","ts":"2026-06-11T10:17:40.849Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e3bb352c-c14a-49d2-a3b6-8c77b0f9fb3d","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:43.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"80c267fa-2ae5-44d4-b4a8-f194f9da33e0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:43.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"80c267fa-2ae5-44d4-b4a8-f194f9da33e0","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:43.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"80c267fa-2ae5-44d4-b4a8-f194f9da33e0","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:43.850Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"80c267fa-2ae5-44d4-b4a8-f194f9da33e0"}
{"level":"info","ts":"2026-06-11T10:17:43.850Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"80c267fa-2ae5-44d4-b4a8-f194f9da33e0","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:46.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2d6922d2-db5a-41be-bbc1-d5d0d9916cb5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:46.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2d6922d2-db5a-41be-bbc1-d5d0d9916cb5","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:46.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2d6922d2-db5a-41be-bbc1-d5d0d9916cb5","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:46.850Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2d6922d2-db5a-41be-bbc1-d5d0d9916cb5"}
{"level":"info","ts":"2026-06-11T10:17:46.850Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2d6922d2-db5a-41be-bbc1-d5d0d9916cb5","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:49.851Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bcb1d942-5ddf-42f2-b3ed-8e86e9194625","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:49.851Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bcb1d942-5ddf-42f2-b3ed-8e86e9194625","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:49.851Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bcb1d942-5ddf-42f2-b3ed-8e86e9194625","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:49.851Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bcb1d942-5ddf-42f2-b3ed-8e86e9194625"}
{"level":"info","ts":"2026-06-11T10:17:49.851Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"bcb1d942-5ddf-42f2-b3ed-8e86e9194625","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:52.852Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e4d18b77-c73d-4381-810b-409e1d74b79e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:52.852Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e4d18b77-c73d-4381-810b-409e1d74b79e","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:52.852Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e4d18b77-c73d-4381-810b-409e1d74b79e","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:52.853Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e4d18b77-c73d-4381-810b-409e1d74b79e"}
{"level":"info","ts":"2026-06-11T10:17:52.853Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"e4d18b77-c73d-4381-810b-409e1d74b79e","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:55.854Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"91c9c786-d848-451f-aeb1-f26633103e0a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:55.854Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"91c9c786-d848-451f-aeb1-f26633103e0a","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:55.854Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"91c9c786-d848-451f-aeb1-f26633103e0a","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:55.854Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"91c9c786-d848-451f-aeb1-f26633103e0a"}
{"level":"info","ts":"2026-06-11T10:17:55.854Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"91c9c786-d848-451f-aeb1-f26633103e0a","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:17:58.854Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"afc8840a-09be-4934-92e9-3e979c34af0e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:17:58.854Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"afc8840a-09be-4934-92e9-3e979c34af0e","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:17:58.854Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"afc8840a-09be-4934-92e9-3e979c34af0e","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:17:58.855Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"afc8840a-09be-4934-92e9-3e979c34af0e"}
{"level":"info","ts":"2026-06-11T10:17:58.855Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"afc8840a-09be-4934-92e9-3e979c34af0e","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:01.855Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7e8bc3a0-e0b0-4cb7-9ff4-13963e0474aa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:01.855Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7e8bc3a0-e0b0-4cb7-9ff4-13963e0474aa","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:01.855Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7e8bc3a0-e0b0-4cb7-9ff4-13963e0474aa","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:01.855Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7e8bc3a0-e0b0-4cb7-9ff4-13963e0474aa"}
{"level":"info","ts":"2026-06-11T10:18:01.855Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7e8bc3a0-e0b0-4cb7-9ff4-13963e0474aa","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","podName":"counter-dqqkt-head-rjvd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:04.853Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"d3cb81b7-b799-410c-ab8e-53a65dd9c669","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:04.856Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f7b73c1b-9c7c-4936-b03a-c9ce8fe07ef4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:04.856Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f7b73c1b-9c7c-4936-b03a-c9ce8fe07ef4","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:04.856Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f7b73c1b-9c7c-4936-b03a-c9ce8fe07ef4","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:04.857Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f7b73c1b-9c7c-4936-b03a-c9ce8fe07ef4"}
{"level":"info","ts":"2026-06-11T10:18:04.857Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f7b73c1b-9c7c-4936-b03a-c9ce8fe07ef4","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701"}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701"}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:18:05.857Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"01825b78-92ec-412d-98e6-335b1e86b701","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:16:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:05.873Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"24166570-381d-4cf1-a714-a2c6f31cf93b","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:05.873Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"d354e671-c88b-401a-81bf-1284b1496b92","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:05.873Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"24166570-381d-4cf1-a714-a2c6f31cf93b","pod":"counter-dqqkt-small-group-worker-shz5x","ip":"10.134.0.31"}
{"level":"info","ts":"2026-06-11T10:18:05.873Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"24166570-381d-4cf1-a714-a2c6f31cf93b","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:18:05.873Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"24166570-381d-4cf1-a714-a2c6f31cf93b","rayCluster":"counter-dqqkt","podIPs":["10.134.0.31","10.133.0.24"]}
{"level":"info","ts":"2026-06-11T10:18:05.873Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"46632f09-22b5-4850-a425-49f875ee1042","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:05.873Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"46632f09-22b5-4850-a425-49f875ee1042","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:05.874Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"46632f09-22b5-4850-a425-49f875ee1042","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:05.874Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"46632f09-22b5-4850-a425-49f875ee1042"}
{"level":"info","ts":"2026-06-11T10:18:05.874Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"46632f09-22b5-4850-a425-49f875ee1042","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:05.893Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"d354e671-c88b-401a-81bf-1284b1496b92","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:18:05.910Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"d354e671-c88b-401a-81bf-1284b1496b92","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:18:05.910Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"d354e671-c88b-401a-81bf-1284b1496b92","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:06.285Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"ec4e4e56-b26f-4146-abd2-9fecdfe3e7ca","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:06.285Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"ec4e4e56-b26f-4146-abd2-9fecdfe3e7ca","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:18:06.285Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"ec4e4e56-b26f-4146-abd2-9fecdfe3e7ca","pod":"counter-dqqkt-small-group-worker-shz5x","ip":"10.134.0.31"}
{"level":"info","ts":"2026-06-11T10:18:06.285Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"ec4e4e56-b26f-4146-abd2-9fecdfe3e7ca","rayCluster":"counter-dqqkt","podIPs":["10.133.0.24","10.134.0.31"]}
{"level":"info","ts":"2026-06-11T10:18:07.857Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"b02ad454-8a10-492c-890b-1002976aaaa8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:07.857Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"b02ad454-8a10-492c-890b-1002976aaaa8","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:07.857Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"b02ad454-8a10-492c-890b-1002976aaaa8","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:07.857Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"b02ad454-8a10-492c-890b-1002976aaaa8"}
{"level":"info","ts":"2026-06-11T10:18:07.857Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"b02ad454-8a10-492c-890b-1002976aaaa8","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811"}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811"}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:07.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:07.876Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:07.876Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:07.876Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:07.876Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"dff8992a-8dc2-4c78-8502-02820b68a811","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47"}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47"}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:09.207Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:09.208Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:09.208Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9bbfd222-8cc7-476c-8129-bd66c3094b47","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:10.208Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d"}
{"level":"info","ts":"2026-06-11T10:18:10.208Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d"}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:18:10.209Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"31308a77-2b6c-448d-b1c1-479c1d49378d","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"2c872748-5e52-4244-9f60-fbeac31bc920","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0e23f62-a6ca-4a99-9466-326a651280ac","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0e23f62-a6ca-4a99-9466-326a651280ac","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0e23f62-a6ca-4a99-9466-326a651280ac","pod":"counter-dqqkt-small-group-worker-shz5x","ip":"10.134.0.31"}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ab1c5904-b0d7-4fc0-9c11-db3d88bbc6df","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ab1c5904-b0d7-4fc0-9c11-db3d88bbc6df","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0e23f62-a6ca-4a99-9466-326a651280ac","rayCluster":"counter-dqqkt","podIPs":["10.133.0.24","10.134.0.31"]}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ab1c5904-b0d7-4fc0-9c11-db3d88bbc6df","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ab1c5904-b0d7-4fc0-9c11-db3d88bbc6df"}
{"level":"info","ts":"2026-06-11T10:18:10.223Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ab1c5904-b0d7-4fc0-9c11-db3d88bbc6df","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:10.234Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"2c872748-5e52-4244-9f60-fbeac31bc920","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:18:10.246Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"2c872748-5e52-4244-9f60-fbeac31bc920","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:18:10.246Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"2c872748-5e52-4244-9f60-fbeac31bc920","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:10.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2b7108e8-5655-417e-a8a3-d108bed1f920","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:10.858Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2b7108e8-5655-417e-a8a3-d108bed1f920","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:10.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2b7108e8-5655-417e-a8a3-d108bed1f920","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:10.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2b7108e8-5655-417e-a8a3-d108bed1f920"}
{"level":"info","ts":"2026-06-11T10:18:10.858Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"2b7108e8-5655-417e-a8a3-d108bed1f920","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3"}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3"}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","podName":"counter-dqqkt-head-rjvd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:12.223Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f1d44d9-fe94-4f6a-94b4-f4e974336ee3","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:13.859Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"dd9dcbf9-7d74-4b0c-aabf-630e704c28b5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:13.859Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"dd9dcbf9-7d74-4b0c-aabf-630e704c28b5","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:13.859Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"dd9dcbf9-7d74-4b0c-aabf-630e704c28b5","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:13.859Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"dd9dcbf9-7d74-4b0c-aabf-630e704c28b5"}
{"level":"info","ts":"2026-06-11T10:18:13.859Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"dd9dcbf9-7d74-4b0c-aabf-630e704c28b5","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:16.584Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cdc21628-c4c0-4058-852d-19c46eea84a3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:18:16.584Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cdc21628-c4c0-4058-852d-19c46eea84a3","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:18:16.585Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cdc21628-c4c0-4058-852d-19c46eea84a3","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:18:16.585Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"cdc21628-c4c0-4058-852d-19c46eea84a3","rayCluster":"raycluster","podIPs":["10.134.0.25","10.132.0.27"]}
{"level":"info","ts":"2026-06-11T10:18:16.860Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d939de67-5718-4e5e-8f4b-e53ec3440980","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:16.860Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d939de67-5718-4e5e-8f4b-e53ec3440980","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:16.860Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d939de67-5718-4e5e-8f4b-e53ec3440980","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:16.861Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d939de67-5718-4e5e-8f4b-e53ec3440980"}
{"level":"info","ts":"2026-06-11T10:18:16.861Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d939de67-5718-4e5e-8f4b-e53ec3440980","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26"}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26"}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","podName":"counter-dqqkt-small-group-worker-shz5x","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","podName":"counter-dqqkt-small-group-worker-shz5x","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:18:16.928Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"915dc658-901f-4a00-a3f9-e6cdec77ae26","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4ea3fd1b-09e5-40fd-bd3e-07ce65de0a71","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"15b7bb28-2ebf-4ca3-abbc-a78561028740","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4ea3fd1b-09e5-40fd-bd3e-07ce65de0a71","pod":"counter-dqqkt-small-group-worker-shz5x","ip":"10.134.0.31"}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4ea3fd1b-09e5-40fd-bd3e-07ce65de0a71","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"59c4d006-12ad-412e-9139-7079d3d80745","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"59c4d006-12ad-412e-9139-7079d3d80745","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4ea3fd1b-09e5-40fd-bd3e-07ce65de0a71","rayCluster":"counter-dqqkt","podIPs":["10.134.0.31","10.133.0.24"]}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"59c4d006-12ad-412e-9139-7079d3d80745","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"59c4d006-12ad-412e-9139-7079d3d80745"}
{"level":"info","ts":"2026-06-11T10:18:16.948Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"59c4d006-12ad-412e-9139-7079d3d80745","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:16.963Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"15b7bb28-2ebf-4ca3-abbc-a78561028740","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:18:16.982Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"15b7bb28-2ebf-4ca3-abbc-a78561028740","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:18:16.982Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"15b7bb28-2ebf-4ca3-abbc-a78561028740","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be"}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be"}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","podName":"counter-dqqkt-small-group-worker-shz5x","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","podName":"counter-dqqkt-small-group-worker-shz5x","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:16:38Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:18.948Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"56f432ae-8cc1-459e-b1de-88481cd187be","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:19.861Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"443c7952-c812-4fb8-86ca-7960938a974a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:19.861Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"443c7952-c812-4fb8-86ca-7960938a974a","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:19.861Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"443c7952-c812-4fb8-86ca-7960938a974a","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:19.861Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"443c7952-c812-4fb8-86ca-7960938a974a"}
{"level":"info","ts":"2026-06-11T10:18:19.861Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"443c7952-c812-4fb8-86ca-7960938a974a","RayCluster":"counter-dqqkt","State":""}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2"}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2"}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:18:21.240Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1c813fe7-0711-4b1f-9d57-76596af48fa2","name":"counter-dqqkt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a6c6ae32-1f3f-4f9c-b559-9bfdaa852b36","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0064685e-4cc9-4b92-9a9b-ffb884480f1f","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a6c6ae32-1f3f-4f9c-b559-9bfdaa852b36","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a6c6ae32-1f3f-4f9c-b559-9bfdaa852b36","pod":"counter-dqqkt-small-group-worker-shz5x","ip":"10.134.0.31"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"a6c6ae32-1f3f-4f9c-b559-9bfdaa852b36","rayCluster":"counter-dqqkt","podIPs":["10.133.0.24","10.134.0.31"]}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","head service name":"counter-dqqkt-head-svc","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","head service URL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700"}
{"level":"info","ts":"2026-06-11T10:18:21.256Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","command":["if","!","ray","job","status","--address","http://counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","counter-thbxx",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-thbxx","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","--follow","counter-thbxx"]}
{"level":"info","ts":"2026-06-11T10:18:21.267Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-11T10:18:21.267Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","SubmissionMode":"K8sJobMode","RayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.267Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:18:21.267Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ebabecae-f738-4121-86d6-c4db95478700","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:18:21.271Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0064685e-4cc9-4b92-9a9b-ffb884480f1f","name":"counter-dqqkt-head"}
{"level":"info","ts":"2026-06-11T10:18:21.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"26454279-5ccc-42bc-8e7c-0f3826d09388","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"26454279-5ccc-42bc-8e7c-0f3826d09388","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"26454279-5ccc-42bc-8e7c-0f3826d09388","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"26454279-5ccc-42bc-8e7c-0f3826d09388"}
{"level":"info","ts":"2026-06-11T10:18:21.286Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0064685e-4cc9-4b92-9a9b-ffb884480f1f","name":"counter-dqqkt-workers"}
{"level":"info","ts":"2026-06-11T10:18:21.286Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"0064685e-4cc9-4b92-9a9b-ffb884480f1f","cluster":"counter-dqqkt"}
{"level":"error","ts":"2026-06-11T10:18:21.292Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"26454279-5ccc-42bc-8e7c-0f3826d09388","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.292Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"26454279-5ccc-42bc-8e7c-0f3826d09388"}
{"level":"error","ts":"2026-06-11T10:18:21.292Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"26454279-5ccc-42bc-8e7c-0f3826d09388","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d5ab2273-8fad-43a2-88bf-08c01597e437","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d5ab2273-8fad-43a2-88bf-08c01597e437","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.292Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d5ab2273-8fad-43a2-88bf-08c01597e437","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.292Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d5ab2273-8fad-43a2-88bf-08c01597e437"}
{"level":"error","ts":"2026-06-11T10:18:21.297Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d5ab2273-8fad-43a2-88bf-08c01597e437","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.297Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d5ab2273-8fad-43a2-88bf-08c01597e437"}
{"level":"error","ts":"2026-06-11T10:18:21.297Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"d5ab2273-8fad-43a2-88bf-08c01597e437","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f2d7dede-4ffd-44fc-a1de-199b4159683a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f2d7dede-4ffd-44fc-a1de-199b4159683a","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f2d7dede-4ffd-44fc-a1de-199b4159683a","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f2d7dede-4ffd-44fc-a1de-199b4159683a"}
{"level":"error","ts":"2026-06-11T10:18:21.332Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f2d7dede-4ffd-44fc-a1de-199b4159683a","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.332Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f2d7dede-4ffd-44fc-a1de-199b4159683a"}
{"level":"error","ts":"2026-06-11T10:18:21.332Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"f2d7dede-4ffd-44fc-a1de-199b4159683a","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.332Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3119d4de-977a-43d7-aa08-755e00ffb07c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3119d4de-977a-43d7-aa08-755e00ffb07c","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3119d4de-977a-43d7-aa08-755e00ffb07c","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3119d4de-977a-43d7-aa08-755e00ffb07c"}
{"level":"error","ts":"2026-06-11T10:18:21.338Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3119d4de-977a-43d7-aa08-755e00ffb07c","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.338Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3119d4de-977a-43d7-aa08-755e00ffb07c"}
{"level":"error","ts":"2026-06-11T10:18:21.338Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"3119d4de-977a-43d7-aa08-755e00ffb07c","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.353Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"44beb5ac-ccc8-4a6c-b2db-14a30b74dba9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.353Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"44beb5ac-ccc8-4a6c-b2db-14a30b74dba9","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.353Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"44beb5ac-ccc8-4a6c-b2db-14a30b74dba9","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.353Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"44beb5ac-ccc8-4a6c-b2db-14a30b74dba9"}
{"level":"error","ts":"2026-06-11T10:18:21.358Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"44beb5ac-ccc8-4a6c-b2db-14a30b74dba9","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.358Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"44beb5ac-ccc8-4a6c-b2db-14a30b74dba9"}
{"level":"error","ts":"2026-06-11T10:18:21.358Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"44beb5ac-ccc8-4a6c-b2db-14a30b74dba9","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.439Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4697f323-abdf-4594-977a-c9101ce5a1e2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.439Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4697f323-abdf-4594-977a-c9101ce5a1e2","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.439Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4697f323-abdf-4594-977a-c9101ce5a1e2","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.439Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4697f323-abdf-4594-977a-c9101ce5a1e2"}
{"level":"error","ts":"2026-06-11T10:18:21.444Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4697f323-abdf-4594-977a-c9101ce5a1e2","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.444Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4697f323-abdf-4594-977a-c9101ce5a1e2"}
{"level":"error","ts":"2026-06-11T10:18:21.444Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4697f323-abdf-4594-977a-c9101ce5a1e2","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"986bd783-ce49-4bad-a98c-9728b9e13843","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"986bd783-ce49-4bad-a98c-9728b9e13843","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.605Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"986bd783-ce49-4bad-a98c-9728b9e13843","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"986bd783-ce49-4bad-a98c-9728b9e13843"}
{"level":"error","ts":"2026-06-11T10:18:21.610Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"986bd783-ce49-4bad-a98c-9728b9e13843","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.610Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"986bd783-ce49-4bad-a98c-9728b9e13843"}
{"level":"error","ts":"2026-06-11T10:18:21.610Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"986bd783-ce49-4bad-a98c-9728b9e13843","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"537403f8-c20f-401e-b383-23e379251fa5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:21.931Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"537403f8-c20f-401e-b383-23e379251fa5","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:21.931Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"537403f8-c20f-401e-b383-23e379251fa5","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:21.931Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"537403f8-c20f-401e-b383-23e379251fa5"}
{"level":"error","ts":"2026-06-11T10:18:21.937Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"537403f8-c20f-401e-b383-23e379251fa5","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:21.937Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"537403f8-c20f-401e-b383-23e379251fa5"}
{"level":"error","ts":"2026-06-11T10:18:21.937Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"537403f8-c20f-401e-b383-23e379251fa5","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:22.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ea48eb54-3783-49a7-a431-62ea395a9f6f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:22.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ea48eb54-3783-49a7-a431-62ea395a9f6f","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:22.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ea48eb54-3783-49a7-a431-62ea395a9f6f","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:22.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ea48eb54-3783-49a7-a431-62ea395a9f6f"}
{"level":"error","ts":"2026-06-11T10:18:22.305Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ea48eb54-3783-49a7-a431-62ea395a9f6f","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:22.305Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ea48eb54-3783-49a7-a431-62ea395a9f6f"}
{"level":"error","ts":"2026-06-11T10:18:22.305Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ea48eb54-3783-49a7-a431-62ea395a9f6f","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:22.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1eaf2132-df12-40de-8f04-9fffedd9a3af","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:22.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1eaf2132-df12-40de-8f04-9fffedd9a3af","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:22.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1eaf2132-df12-40de-8f04-9fffedd9a3af","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:22.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1eaf2132-df12-40de-8f04-9fffedd9a3af"}
{"level":"error","ts":"2026-06-11T10:18:22.583Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1eaf2132-df12-40de-8f04-9fffedd9a3af","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:22.583Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1eaf2132-df12-40de-8f04-9fffedd9a3af"}
{"level":"error","ts":"2026-06-11T10:18:22.583Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1eaf2132-df12-40de-8f04-9fffedd9a3af","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3"}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3"}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","1 head service found":"counter-dqqkt-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","Found 1 head Pod":"counter-dqqkt-head-rjvd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","head Pod":"counter-dqqkt-head-rjvd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-dqqkt-head-rjvd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","worker Pod":"counter-dqqkt-small-group-worker-shz5x","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-dqqkt-small-group-worker-shz5x. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:23.257Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"1f692492-a87a-45e8-b8d5-4844f780c6e3","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:25.143Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"feb6b213-27e3-468b-be48-cb02d8717499","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:25.143Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"feb6b213-27e3-468b-be48-cb02d8717499","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:25.143Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"feb6b213-27e3-468b-be48-cb02d8717499","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:25.144Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"feb6b213-27e3-468b-be48-cb02d8717499"}
{"level":"error","ts":"2026-06-11T10:18:25.149Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"feb6b213-27e3-468b-be48-cb02d8717499","JobId":"counter-thbxx","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:25.149Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"feb6b213-27e3-468b-be48-cb02d8717499"}
{"level":"error","ts":"2026-06-11T10:18:25.149Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"feb6b213-27e3-468b-be48-cb02d8717499","error":"Job counter-thbxx does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:30.270Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"0f69ba04-8f1b-492a-acdd-64db8edf96b9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:30.270Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"0f69ba04-8f1b-492a-acdd-64db8edf96b9","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:30.270Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"0f69ba04-8f1b-492a-acdd-64db8edf96b9","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:30.270Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"0f69ba04-8f1b-492a-acdd-64db8edf96b9"}
{"level":"info","ts":"2026-06-11T10:18:30.283Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"0f69ba04-8f1b-492a-acdd-64db8edf96b9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:30.283Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"0f69ba04-8f1b-492a-acdd-64db8edf96b9","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:18:30.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"829e3c75-9b55-4a0d-9b77-bf172b5416f6","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:30.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"829e3c75-9b55-4a0d-9b77-bf172b5416f6","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:30.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"829e3c75-9b55-4a0d-9b77-bf172b5416f6","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:30.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"829e3c75-9b55-4a0d-9b77-bf172b5416f6"}
{"level":"info","ts":"2026-06-11T10:18:30.300Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"829e3c75-9b55-4a0d-9b77-bf172b5416f6","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:33.296Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a12555be-c9c0-49c4-9217-23cd67c549eb","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:33.296Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a12555be-c9c0-49c4-9217-23cd67c549eb","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:33.296Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a12555be-c9c0-49c4-9217-23cd67c549eb","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:33.296Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a12555be-c9c0-49c4-9217-23cd67c549eb"}
{"level":"info","ts":"2026-06-11T10:18:33.300Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a12555be-c9c0-49c4-9217-23cd67c549eb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:33.300Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a12555be-c9c0-49c4-9217-23cd67c549eb","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:18:33.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ca533354-f7b8-4cec-872e-fa67970fe003","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:33.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ca533354-f7b8-4cec-872e-fa67970fe003","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:33.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ca533354-f7b8-4cec-872e-fa67970fe003","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:33.316Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ca533354-f7b8-4cec-872e-fa67970fe003"}
{"level":"info","ts":"2026-06-11T10:18:33.319Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"ca533354-f7b8-4cec-872e-fa67970fe003","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:36.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7c224692-3e67-4739-a6ee-248610bdb70d","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:36.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7c224692-3e67-4739-a6ee-248610bdb70d","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:36.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7c224692-3e67-4739-a6ee-248610bdb70d","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:36.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7c224692-3e67-4739-a6ee-248610bdb70d"}
{"level":"info","ts":"2026-06-11T10:18:36.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7c224692-3e67-4739-a6ee-248610bdb70d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:39.319Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"90046510-123f-49d6-b8a4-defa700b19a4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:39.319Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"90046510-123f-49d6-b8a4-defa700b19a4","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:39.319Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"90046510-123f-49d6-b8a4-defa700b19a4","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:39.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"90046510-123f-49d6-b8a4-defa700b19a4"}
{"level":"info","ts":"2026-06-11T10:18:39.322Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"90046510-123f-49d6-b8a4-defa700b19a4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:39.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"159f3e83-4b66-412d-99ef-312441f473a8","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:39.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"159f3e83-4b66-412d-99ef-312441f473a8","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:39.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"159f3e83-4b66-412d-99ef-312441f473a8","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:39.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"159f3e83-4b66-412d-99ef-312441f473a8"}
{"level":"info","ts":"2026-06-11T10:18:39.348Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"159f3e83-4b66-412d-99ef-312441f473a8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:40.356Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1ef5541a-ee5a-4f5d-9219-9ecfd4ed9364","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:40.356Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1ef5541a-ee5a-4f5d-9219-9ecfd4ed9364","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.356Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1ef5541a-ee5a-4f5d-9219-9ecfd4ed9364","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.356Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1ef5541a-ee5a-4f5d-9219-9ecfd4ed9364"}
{"level":"info","ts":"2026-06-11T10:18:40.359Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"1ef5541a-ee5a-4f5d-9219-9ecfd4ed9364","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:40.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"32e5bf52-1f8e-4ace-8e00-52b62b76ec85","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:40.376Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"32e5bf52-1f8e-4ace-8e00-52b62b76ec85","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"32e5bf52-1f8e-4ace-8e00-52b62b76ec85","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"32e5bf52-1f8e-4ace-8e00-52b62b76ec85"}
{"level":"info","ts":"2026-06-11T10:18:40.379Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"32e5bf52-1f8e-4ace-8e00-52b62b76ec85","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:18:28Z","endTime":"2026-06-11T10:18:32Z"},"jobId":"counter-thbxx","rayClusterName":"counter-dqqkt","dashboardURL":"counter-dqqkt-head-svc.test-ns-v8w6c.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-11T10:16:35Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:21Z","stateTransitionTimes":{"ready":"2026-06-11T10:18:21Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.24","serviceIP":"10.133.0.24","podName":"counter-dqqkt-head-rjvd6","serviceName":"counter-dqqkt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:18:16Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:18:21Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:16:35Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:16:35Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:18:40.379Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"32e5bf52-1f8e-4ace-8e00-52b62b76ec85","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-11T10:18:40.396Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4376cf98-c7b9-43be-99ca-8263870c8e4f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:40.396Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4376cf98-c7b9-43be-99ca-8263870c8e4f","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:18:40 +0000 UTC","Now":"2026-06-11T10:18:40.396Z","ShutdownTime":"2026-06-11T10:18:40.000Z"}
{"level":"info","ts":"2026-06-11T10:18:40.396Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4376cf98-c7b9-43be-99ca-8263870c8e4f","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:18:40.407Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4376cf98-c7b9-43be-99ca-8263870c8e4f","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.407Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4376cf98-c7b9-43be-99ca-8263870c8e4f","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:18:40.407Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"4376cf98-c7b9-43be-99ca-8263870c8e4f","RayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4a00648e-84df-4589-8fa0-9d2cec91f886","rayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"reconcileID":"9b4fb661-4c1c-4405-b6b4-ff72f200b8c8"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"be2d4ce0-de49-497c-b06a-d03c9938d5d0"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","namespace":"test-ns-v8w6c","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4a00648e-84df-4589-8fa0-9d2cec91f886","pod":"counter-dqqkt-head-rjvd6","ip":"10.133.0.24"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4a00648e-84df-4589-8fa0-9d2cec91f886","pod":"counter-dqqkt-small-group-worker-shz5x","ip":"10.134.0.31"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6e6d7685-77c2-4f62-8a25-f82ac5731705","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6e6d7685-77c2-4f62-8a25-f82ac5731705","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:18:40 +0000 UTC","Now":"2026-06-11T10:18:40.408Z","ShutdownTime":"2026-06-11T10:18:40.000Z"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6e6d7685-77c2-4f62-8a25-f82ac5731705","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6e6d7685-77c2-4f62-8a25-f82ac5731705","RayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6e6d7685-77c2-4f62-8a25-f82ac5731705","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6e6d7685-77c2-4f62-8a25-f82ac5731705","RayCluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.408Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"4a00648e-84df-4589-8fa0-9d2cec91f886","rayCluster":"counter-dqqkt","podIPs":["10.133.0.24","10.134.0.31"]}
{"level":"info","ts":"2026-06-11T10:18:40.413Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","name":"kuberay-gateway-access","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:18:40.419Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","name":"test-ns-v8w6c-counter-dqqkt","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:18:40.426Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","configMap":"kube-rbac-proxy-config-counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.435Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","serviceAccount":"counter-dqqkt-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:18:40.458Z","logger":"raycluster-resource","msg":"default","name":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.458Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-dqqkt","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"05f777a3-e069-4e0b-bf73-ae272e8fff82","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b","cluster":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b","namespace":"test-ns-v8w6c","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"e9801ca3-67ce-409e-9d6a-2e30c0e19a47"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"38c851f3-bca9-44e2-9771-b52ecdeab278","namespace":"test-ns-v8w6c","clusterName":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a67834a4-dbd0-4c61-b77b-ae2f304d4e82","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a67834a4-dbd0-4c61-b77b-ae2f304d4e82","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:18:40 +0000 UTC","Now":"2026-06-11T10:18:40.470Z","ShutdownTime":"2026-06-11T10:18:40.000Z"}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a67834a4-dbd0-4c61-b77b-ae2f304d4e82","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a67834a4-dbd0-4c61-b77b-ae2f304d4e82","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a67834a4-dbd0-4c61-b77b-ae2f304d4e82","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:18:40.470Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"a67834a4-dbd0-4c61-b77b-ae2f304d4e82","RayCluster":"counter-dqqkt"}
{"level":"error","ts":"2026-06-11T10:18:40.474Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b","error":"rayclusters.ray.io \"counter-dqqkt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:40.474Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b"}
{"level":"error","ts":"2026-06-11T10:18:40.474Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"c0dfc038-e9e7-4e55-b18f-b45fbbc4363b","error":"rayclusters.ray.io \"counter-dqqkt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:40.474Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dc377807-1333-4277-8999-019522abcc1f","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.474Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dc377807-1333-4277-8999-019522abcc1f"}
{"level":"info","ts":"2026-06-11T10:18:40.474Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dc377807-1333-4277-8999-019522abcc1f","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:18:40.474Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"dc377807-1333-4277-8999-019522abcc1f","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:18:40.480Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b4beb1af-0e74-46f5-af34-5f0beba7fc38","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.480Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b4beb1af-0e74-46f5-af34-5f0beba7fc38"}
{"level":"info","ts":"2026-06-11T10:18:40.480Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b4beb1af-0e74-46f5-af34-5f0beba7fc38","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:18:40.480Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"b4beb1af-0e74-46f5-af34-5f0beba7fc38","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:18:40.480Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"51dd13a3-67e8-44b6-acaa-6e7643c987b7"}
{"level":"info","ts":"2026-06-11T10:18:40.482Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"13f5825b-03c0-4bce-977f-f9591b870875","namespacedName":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"}}
{"level":"info","ts":"2026-06-11T10:18:40.482Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"13f5825b-03c0-4bce-977f-f9591b870875"}
{"level":"info","ts":"2026-06-11T10:18:40.482Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"13f5825b-03c0-4bce-977f-f9591b870875","namespace":"test-ns-v8w6c","authClusters":0}
{"level":"info","ts":"2026-06-11T10:18:40.482Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"13f5825b-03c0-4bce-977f-f9591b870875","namespace":"test-ns-v8w6c"}
{"level":"info","ts":"2026-06-11T10:18:40.502Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"5f63d739-b36b-4c1d-a74d-ff2488083eb6"}
{"level":"error","ts":"2026-06-11T10:18:40.519Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"38c851f3-bca9-44e2-9771-b52ecdeab278","certificate":"ray-ca-certificate-counter-dqqkt","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-dqqkt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:40.538Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"38c851f3-bca9-44e2-9771-b52ecdeab278","errorCount":1}
{"level":"info","ts":"2026-06-11T10:18:40.538Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"38c851f3-bca9-44e2-9771-b52ecdeab278"}
{"level":"error","ts":"2026-06-11T10:18:40.538Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"38c851f3-bca9-44e2-9771-b52ecdeab278","error":"certificates.cert-manager.io \"ray-ca-certificate-counter-dqqkt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:40.543Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"e34a1528-a778-40d0-aacc-553c59e44c6f","namespace":"test-ns-v8w6c","clusterName":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.543Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-dqqkt","namespace":"test-ns-v8w6c"},"namespace":"test-ns-v8w6c","name":"counter-dqqkt","reconcileID":"e34a1528-a778-40d0-aacc-553c59e44c6f","namespace":"test-ns-v8w6c","clusterName":"counter-dqqkt"}
{"level":"info","ts":"2026-06-11T10:18:40.651Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6b2fe4b7-a9be-4543-a30e-e3e83e3e2ac2","DeletionTimestamp":"2026-06-11 10:18:40 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:18:40.651Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"6b2fe4b7-a9be-4543-a30e-e3e83e3e2ac2","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:18:40.667Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"7948f1aa-5f3d-4720-9510-5b47a59a1ee1"}
{"level":"info","ts":"2026-06-11T10:18:40.675Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"c46309ab-25f8-44dc-8284-d60450f1d87a"}
{"level":"info","ts":"2026-06-11T10:18:42.323Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-v8w6c"},"reconcileID":"b2274b03-f3c3-442b-8697-fdebcac6ddf1"}
{"level":"info","ts":"2026-06-11T10:18:42.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"aed570df-b9e1-4606-9329-0ee46304d1df","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:42.800Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"aed570df-b9e1-4606-9329-0ee46304d1df","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:18:42.817Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"aed570df-b9e1-4606-9329-0ee46304d1df"}
{"level":"info","ts":"2026-06-11T10:18:42.817Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"aed570df-b9e1-4606-9329-0ee46304d1df","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:18:42.817Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"aed570df-b9e1-4606-9329-0ee46304d1df","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:18:42.817Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"aed570df-b9e1-4606-9329-0ee46304d1df","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:18:42.839Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:42.839Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9"}
{"level":"info","ts":"2026-06-11T10:18:42.839Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:18:42.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p485k","rayClusterName":"counter-h4stj","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:18:42.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:18:42.847Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:18:42.847Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9"}
{"level":"error","ts":"2026-06-11T10:18:42.847Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7fd48a37-edf4-452d-b88c-5b88c11dace9","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ab5fb0be-1d1a-4209-a82c-26f7bcc28ae2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:42.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ab5fb0be-1d1a-4209-a82c-26f7bcc28ae2","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.847Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ab5fb0be-1d1a-4209-a82c-26f7bcc28ae2","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.847Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ab5fb0be-1d1a-4209-a82c-26f7bcc28ae2","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.852Z","logger":"raycluster-resource","msg":"default","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.852Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-9qkzh","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ab5fb0be-1d1a-4209-a82c-26f7bcc28ae2","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ab5fb0be-1d1a-4209-a82c-26f7bcc28ae2"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ab5fb0be-1d1a-4209-a82c-26f7bcc28ae2","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"49d14ad5-fe80-4cbf-a14e-f6dddc7b9c24","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"46734a25-c5e9-4803-a5f1-54b448e22f6f","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"49d14ad5-fe80-4cbf-a14e-f6dddc7b9c24","mode":"IntegratedOAuth","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"49d14ad5-fe80-4cbf-a14e-f6dddc7b9c24","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"49d14ad5-fe80-4cbf-a14e-f6dddc7b9c24","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ebeb4684-449f-463d-ab88-1fc267b5d10c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ebeb4684-449f-463d-ab88-1fc267b5d10c","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ebeb4684-449f-463d-ab88-1fc267b5d10c","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ebeb4684-449f-463d-ab88-1fc267b5d10c"}
{"level":"info","ts":"2026-06-11T10:18:42.913Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ebeb4684-449f-463d-ab88-1fc267b5d10c","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:42.914Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b5d9fe9b-5422-42c3-9ace-34bcc4ec1381","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:42.914Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b5d9fe9b-5422-42c3-9ace-34bcc4ec1381","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.914Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b5d9fe9b-5422-42c3-9ace-34bcc4ec1381","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.914Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b5d9fe9b-5422-42c3-9ace-34bcc4ec1381"}
{"level":"info","ts":"2026-06-11T10:18:42.914Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b5d9fe9b-5422-42c3-9ace-34bcc4ec1381","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","name":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","serviceSelectors":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/identifier":"counter-9qkzh-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","error":"unable to find head service. cluster name counter-9qkzh, filter labels [test-ns-c6srn map[ray.io/cluster:counter-9qkzh ray.io/identifier:counter-9qkzh-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a"}
{"level":"error","ts":"2026-06-11T10:18:42.925Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a813848b-814d-4225-8de1-16763c14ee1a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853"}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853"}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:18:42.926Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","name":"counter-9qkzh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:42.956Z","logger":"raycluster-resource","msg":"default","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.956Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-9qkzh","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853"}
{"level":"error","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d34b2823-5915-4a52-a70a-071732c28853","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8"}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8"}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:18:42.973Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","name":"counter-9qkzh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:42.985Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cadc143b-6edc-4ea7-8ca3-a7adabff3a68","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:42.985Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cadc143b-6edc-4ea7-8ca3-a7adabff3a68","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.985Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cadc143b-6edc-4ea7-8ca3-a7adabff3a68","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.985Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cadc143b-6edc-4ea7-8ca3-a7adabff3a68"}
{"level":"info","ts":"2026-06-11T10:18:42.985Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cadc143b-6edc-4ea7-8ca3-a7adabff3a68","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:42.986Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"46734a25-c5e9-4803-a5f1-54b448e22f6f","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.986Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"70c4c71b-bd44-43ec-944b-2628bca0ae1b","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.991Z","logger":"raycluster-resource","msg":"default","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.991Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-9qkzh","namespace":"test-ns-c6srn"}
{"level":"error","ts":"2026-06-11T10:18:42.996Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"49d14ad5-fe80-4cbf-a14e-f6dddc7b9c24","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-9qkzh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.996Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"49d14ad5-fe80-4cbf-a14e-f6dddc7b9c24"}
{"level":"error","ts":"2026-06-11T10:18:42.996Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"49d14ad5-fe80-4cbf-a14e-f6dddc7b9c24","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-9qkzh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:42.996Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:42.996Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","mode":"IntegratedOAuth","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.996Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:42.996Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.010Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"70c4c71b-bd44-43ec-944b-2628bca0ae1b","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:18:43.019Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","name":"counter-9qkzh","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-9qkzh\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"counter-9qkzh"}}
{"level":"info","ts":"2026-06-11T10:18:43.019Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8"}
{"level":"error","ts":"2026-06-11T10:18:43.019Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6bd5d3fe-8a75-4a4b-90e0-6b74e7835fe8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2"}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2"}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2"}
{"level":"error","ts":"2026-06-11T10:18:43.020Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6a2c8bd8-9ad7-4a6d-943e-76a30e76a5b2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.025Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"70c4c71b-bd44-43ec-944b-2628bca0ae1b","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:18:43.025Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"70c4c71b-bd44-43ec-944b-2628bca0ae1b","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.031Z","logger":"raycluster-resource","msg":"default","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.031Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-9qkzh","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6"}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6"}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6"}
{"level":"error","ts":"2026-06-11T10:18:43.040Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"5b1adee8-ac45-4a9c-9c71-6bc50f0cc8c6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.046Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.046Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.075Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"db02b7e1-aacd-4712-847b-9291d779bfec","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.076Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"d9617e71-c27b-4280-99f4-55fbb7484f2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:43.076Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"d9617e71-c27b-4280-99f4-55fbb7484f2a","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.076Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"d9617e71-c27b-4280-99f4-55fbb7484f2a","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:43.076Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"d9617e71-c27b-4280-99f4-55fbb7484f2a"}
{"level":"info","ts":"2026-06-11T10:18:43.076Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"d9617e71-c27b-4280-99f4-55fbb7484f2a","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:43.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98"}
{"level":"info","ts":"2026-06-11T10:18:43.127Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98"}
{"level":"info","ts":"2026-06-11T10:18:43.127Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","name":"counter-9qkzh-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:18:43.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:43.127Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b"}
{"level":"error","ts":"2026-06-11T10:18:43.227Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:43.135Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"db02b7e1-aacd-4712-847b-9291d779bfec","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.227Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98"}
{"level":"error","ts":"2026-06-11T10:18:43.228Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6e7958b5-deeb-496f-bf7f-13ff0c5ffc98","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.328Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.328Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","name":"kuberay-gateway-access","namespace":"test-ns-c6srn","operation":"created"}
{"level":"info","ts":"2026-06-11T10:18:43.334Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"db02b7e1-aacd-4712-847b-9291d779bfec","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:18:43.334Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"db02b7e1-aacd-4712-847b-9291d779bfec","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.341Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","name":"test-ns-c6srn-counter-9qkzh","operation":"created"}
{"level":"info","ts":"2026-06-11T10:18:43.347Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b"}
{"level":"info","ts":"2026-06-11T10:18:43.347Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:18:43.354Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","name":"kube-rbac-proxy-config-counter-9qkzh","operation":"created"}
{"level":"info","ts":"2026-06-11T10:18:43.372Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b"}
{"level":"info","ts":"2026-06-11T10:18:43.372Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b","rayCluster":"counter-9qkzh","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:18:43.372Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"0eb6ded9-b235-4223-8a77-ed4a44ddd18b"}
{"level":"info","ts":"2026-06-11T10:18:43.372Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"1868810d-404e-47e6-8f43-6a858b5507d3","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.373Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"1868810d-404e-47e6-8f43-6a858b5507d3"}
{"level":"info","ts":"2026-06-11T10:18:43.373Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"c02e113f-886b-42ac-9162-55a941eed7a8","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.373Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"652f6e84-453f-4ab3-967d-ac71f886f730","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:43.373Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"652f6e84-453f-4ab3-967d-ac71f886f730","mode":"IntegratedOAuth","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.373Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"652f6e84-453f-4ab3-967d-ac71f886f730","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.374Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2342030f-1871-4b3a-ba92-4963aae372fd","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.374Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"de997199-c528-4099-8f68-aa0a6a2798e8","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.374Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"de997199-c528-4099-8f68-aa0a6a2798e8"}
{"level":"info","ts":"2026-06-11T10:18:43.374Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c4794c1b-8499-42ed-b9d9-5569c5d91ed3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:43.374Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c4794c1b-8499-42ed-b9d9-5569c5d91ed3","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.374Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c4794c1b-8499-42ed-b9d9-5569c5d91ed3","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:43.375Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c4794c1b-8499-42ed-b9d9-5569c5d91ed3"}
{"level":"info","ts":"2026-06-11T10:18:43.375Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c4794c1b-8499-42ed-b9d9-5569c5d91ed3","RayCluster":"counter-9qkzh","State":""}
{"level":"error","ts":"2026-06-11T10:18:43.382Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"652f6e84-453f-4ab3-967d-ac71f886f730","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-9qkzh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.382Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"652f6e84-453f-4ab3-967d-ac71f886f730"}
{"level":"error","ts":"2026-06-11T10:18:43.382Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"652f6e84-453f-4ab3-967d-ac71f886f730","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-9qkzh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.388Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"e086c958-c7b9-4fc2-8d69-9b6108c8cfab","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:43.388Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"e086c958-c7b9-4fc2-8d69-9b6108c8cfab","mode":"IntegratedOAuth","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.388Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"e086c958-c7b9-4fc2-8d69-9b6108c8cfab","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a"}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a"}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a"}
{"level":"error","ts":"2026-06-11T10:18:43.427Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"35643955-a10f-4dcb-91d7-361d7adb451a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.435Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2342030f-1871-4b3a-ba92-4963aae372fd","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:18:43.443Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"e086c958-c7b9-4fc2-8d69-9b6108c8cfab","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.449Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2342030f-1871-4b3a-ba92-4963aae372fd","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:18:43.449Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2342030f-1871-4b3a-ba92-4963aae372fd","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134"}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134"}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134"}
{"level":"error","ts":"2026-06-11T10:18:43.748Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"7c8528dd-d974-4c4c-8f4d-92596850f134","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a"}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a"}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","error":"head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"error","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found"}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","filter labels":["test-ns-c6srn",{"ray.io/cluster":"counter-9qkzh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a"}
{"level":"error","ts":"2026-06-11T10:18:44.389Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"ccfec862-085f-4b45-87c3-839c64b9629a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-9qkzh not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","mode":"IntegratedOAuth","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","cluster":"counter-9qkzh","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","cluster":"counter-9qkzh","authType":"IntegratedOAuth","serviceAccount":"counter-9qkzh-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-9qkzh","ray.io/group":"headgroup","ray.io/identifier":"counter-9qkzh-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:18:45.670Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","enabled":true}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","name":"counter-9qkzh-head-xhzgz"}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781"}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","nodeType":"worker","rayStartParams":{"address":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","rayStartCmd":"ray start  --address=counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:18:45.690Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","enabled":true}
{"level":"info","ts":"2026-06-11T10:18:45.712Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","name":"counter-9qkzh-small-group-worker-szjpx"}
{"level":"info","ts":"2026-06-11T10:18:45.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.712Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-9qkzh-head-svc"},"newHead":{"podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"}}
{"level":"info","ts":"2026-06-11T10:18:45.712Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"de3cf3a3-d5d7-4493-a6bc-032c65e0c781","name":"counter-9qkzh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77"}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77"}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"53da6548-7725-4a56-babc-f789cf93868e","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:45.732Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d5772b00-6627-46d2-83b2-9bf567c83357","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f4516287-e1e0-458a-96cb-af52755213ae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f4516287-e1e0-458a-96cb-af52755213ae","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-9qkzh-head-svc"},"newHead":{"podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"}}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f4516287-e1e0-458a-96cb-af52755213ae","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"53da6548-7725-4a56-babc-f789cf93868e","rayCluster":"counter-9qkzh","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","name":"counter-9qkzh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f4516287-e1e0-458a-96cb-af52755213ae"}
{"level":"info","ts":"2026-06-11T10:18:45.733Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f4516287-e1e0-458a-96cb-af52755213ae","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:45.741Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","name":"counter-9qkzh","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-9qkzh\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"counter-9qkzh"}}
{"level":"info","ts":"2026-06-11T10:18:45.741Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77"}
{"level":"error","ts":"2026-06-11T10:18:45.741Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c1da4406-b134-4185-9cef-19cd5a862f77","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-9qkzh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:45.745Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"884d4899-d0f3-4a89-9218-75e1639fbf07","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:45.747Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"20b634dd-9776-4a7a-902e-eeb838fc4958","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:45.749Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d5772b00-6627-46d2-83b2-9bf567c83357","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:18:45.761Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d5772b00-6627-46d2-83b2-9bf567c83357","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:18:45.761Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d5772b00-6627-46d2-83b2-9bf567c83357","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:45.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"44bc0a2d-1e5e-4685-be5b-dc0e250158c4","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:45.914Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"225c0304-d65f-4f4f-8f2a-1eab0288d01c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:45.914Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"225c0304-d65f-4f4f-8f2a-1eab0288d01c","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:45.914Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"225c0304-d65f-4f4f-8f2a-1eab0288d01c","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:45.914Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"225c0304-d65f-4f4f-8f2a-1eab0288d01c"}
{"level":"info","ts":"2026-06-11T10:18:45.914Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"225c0304-d65f-4f4f-8f2a-1eab0288d01c","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466"}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466"}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:46.181Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:46.182Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:46.182Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:46.182Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"1a0c8a2a-a04a-46af-b5c1-2baf91db8466","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:46.376Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"2b348a39-6b27-4051-a1ea-bb2e80d6ace4","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d"}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d"}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:47.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:47.661Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:47.661Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"3eb802df-75d2-413b-ad69-cbac5fa09c3d","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"newHead":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"}}
{"level":"info","ts":"2026-06-11T10:18:48.012Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"39e7816c-9237-4280-b324-7cc597b5a8a3","name":"counter-9qkzh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4afa1a64-aabf-44dd-91d1-1b0e9cad0144","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"bba6ee4d-703c-4e24-9213-27fb0c825253","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4afa1a64-aabf-44dd-91d1-1b0e9cad0144","pod":"counter-9qkzh-head-xhzgz","ip":"10.133.0.25"}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4afa1a64-aabf-44dd-91d1-1b0e9cad0144","pod":"counter-9qkzh-small-group-worker-szjpx","ip":"10.134.0.33"}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"eb2e21a5-f53d-4e0e-b10a-f5643e22ca47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"eb2e21a5-f53d-4e0e-b10a-f5643e22ca47","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"eb2e21a5-f53d-4e0e-b10a-f5643e22ca47","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"eb2e21a5-f53d-4e0e-b10a-f5643e22ca47"}
{"level":"info","ts":"2026-06-11T10:18:48.031Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"eb2e21a5-f53d-4e0e-b10a-f5643e22ca47","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:48.044Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"bba6ee4d-703c-4e24-9213-27fb0c825253","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:18:48.045Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4afa1a64-aabf-44dd-91d1-1b0e9cad0144","certificate":"ray-head-cert-counter-9qkzh","dnsNames":["counter-9qkzh-head-svc","localhost","counter-9qkzh-head-svc.test-ns-c6srn.svc","counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.25","10.134.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:18:48.057Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4afa1a64-aabf-44dd-91d1-1b0e9cad0144","certificate":"ray-worker-cert-counter-9qkzh","dnsNames":["counter-9qkzh-worker-svc","localhost","counter-9qkzh-small-group","counter-9qkzh-worker-svc.test-ns-c6srn.svc","counter-9qkzh-worker-svc.test-ns-c6srn.svc.cluster.local","*.counter-9qkzh-worker-svc.test-ns-c6srn.svc","*.counter-9qkzh-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.25","10.134.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:18:48.058Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4afa1a64-aabf-44dd-91d1-1b0e9cad0144","rayCluster":"counter-9qkzh","podIPs":["10.133.0.25","10.134.0.33"]}
{"level":"info","ts":"2026-06-11T10:18:48.060Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"bba6ee4d-703c-4e24-9213-27fb0c825253","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:18:48.060Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"bba6ee4d-703c-4e24-9213-27fb0c825253","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:48.915Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"8a0fddf0-460d-4553-a602-40f09384e949","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:48.915Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"8a0fddf0-460d-4553-a602-40f09384e949","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:48.915Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"8a0fddf0-460d-4553-a602-40f09384e949","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:48.915Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"8a0fddf0-460d-4553-a602-40f09384e949"}
{"level":"info","ts":"2026-06-11T10:18:48.915Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"8a0fddf0-460d-4553-a602-40f09384e949","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:50.030Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"f3dfbb21-6631-4fa3-84dd-09fa520078d9","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:51.916Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"30665454-a2e6-457c-9576-49ed5c805407","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:51.916Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"30665454-a2e6-457c-9576-49ed5c805407","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:51.916Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"30665454-a2e6-457c-9576-49ed5c805407","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:51.916Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"30665454-a2e6-457c-9576-49ed5c805407"}
{"level":"info","ts":"2026-06-11T10:18:51.916Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"30665454-a2e6-457c-9576-49ed5c805407","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","podName":"counter-9qkzh-head-xhzgz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:53.034Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"c155fece-3f6e-44b7-af99-c627dca3dac0","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588"}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588"}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","podName":"counter-9qkzh-head-xhzgz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:54.043Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","podName":"counter-9qkzh-head-xhzgz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:54.044Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:18:54.044Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b61bdf7-ef69-4b9d-b080-52b57f0e4588","name":"counter-9qkzh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9cfcb3ff-c33f-4331-976e-6f463188f056","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"5ecc66bd-8980-4d05-93ae-44c019c1bf28","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"5ecc66bd-8980-4d05-93ae-44c019c1bf28","pod":"counter-9qkzh-head-xhzgz","ip":"10.133.0.25"}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"5ecc66bd-8980-4d05-93ae-44c019c1bf28","pod":"counter-9qkzh-small-group-worker-szjpx","ip":"10.134.0.33"}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"61fc3919-8466-4e8a-8c21-6f9d1e5eaeac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"61fc3919-8466-4e8a-8c21-6f9d1e5eaeac","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"5ecc66bd-8980-4d05-93ae-44c019c1bf28","rayCluster":"counter-9qkzh","podIPs":["10.133.0.25","10.134.0.33"]}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"61fc3919-8466-4e8a-8c21-6f9d1e5eaeac","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"61fc3919-8466-4e8a-8c21-6f9d1e5eaeac"}
{"level":"info","ts":"2026-06-11T10:18:54.058Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"61fc3919-8466-4e8a-8c21-6f9d1e5eaeac","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:54.071Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9cfcb3ff-c33f-4331-976e-6f463188f056","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:18:54.083Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9cfcb3ff-c33f-4331-976e-6f463188f056","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:18:54.083Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9cfcb3ff-c33f-4331-976e-6f463188f056","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:54.917Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cf14f147-d883-485a-a32d-3350d0d8ccb6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:54.917Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cf14f147-d883-485a-a32d-3350d0d8ccb6","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:54.917Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cf14f147-d883-485a-a32d-3350d0d8ccb6","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:54.917Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cf14f147-d883-485a-a32d-3350d0d8ccb6"}
{"level":"info","ts":"2026-06-11T10:18:54.917Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cf14f147-d883-485a-a32d-3350d0d8ccb6","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1"}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1"}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","podName":"counter-9qkzh-head-xhzgz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","podName":"counter-9qkzh-head-xhzgz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:18:56.059Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"597c4d9e-a43e-42ff-a372-4e7ffe7be4f1","seconds":300}
{"level":"info","ts":"2026-06-11T10:18:57.918Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cee1f96b-348a-4343-9d0a-c49293d66372","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:18:57.918Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cee1f96b-348a-4343-9d0a-c49293d66372","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:18:57.918Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cee1f96b-348a-4343-9d0a-c49293d66372","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:18:57.918Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cee1f96b-348a-4343-9d0a-c49293d66372"}
{"level":"info","ts":"2026-06-11T10:18:57.918Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"cee1f96b-348a-4343-9d0a-c49293d66372","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:00.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ec73c5e9-e57b-4507-8d6c-bc3a4f933e6e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:00.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ec73c5e9-e57b-4507-8d6c-bc3a4f933e6e","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:00.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ec73c5e9-e57b-4507-8d6c-bc3a4f933e6e","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:00.919Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ec73c5e9-e57b-4507-8d6c-bc3a4f933e6e"}
{"level":"info","ts":"2026-06-11T10:19:00.919Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"ec73c5e9-e57b-4507-8d6c-bc3a4f933e6e","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:03.920Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11e02b10-8b85-4a0d-abf0-a507acdf566f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:03.920Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11e02b10-8b85-4a0d-abf0-a507acdf566f","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:03.920Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11e02b10-8b85-4a0d-abf0-a507acdf566f","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:03.920Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11e02b10-8b85-4a0d-abf0-a507acdf566f"}
{"level":"info","ts":"2026-06-11T10:19:03.920Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11e02b10-8b85-4a0d-abf0-a507acdf566f","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:19:05.078Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"86029696-a7b5-4c0b-a5f8-6cb3dc46f648","name":"counter-9qkzh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:18:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:19:05.096Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"514a7f6c-d64c-4dea-a3a1-22b8b557243e","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:05.096Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9e07f2a2-36a0-489f-b938-d2af3b88619b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:05.096Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9e07f2a2-36a0-489f-b938-d2af3b88619b","pod":"counter-9qkzh-head-xhzgz","ip":"10.133.0.25"}
{"level":"info","ts":"2026-06-11T10:19:05.096Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"a1777b73-8702-47b0-931e-a9f0fd9876be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:05.097Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9e07f2a2-36a0-489f-b938-d2af3b88619b","pod":"counter-9qkzh-small-group-worker-szjpx","ip":"10.134.0.33"}
{"level":"info","ts":"2026-06-11T10:19:05.097Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"a1777b73-8702-47b0-931e-a9f0fd9876be","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:05.097Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"a1777b73-8702-47b0-931e-a9f0fd9876be","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:05.097Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"9e07f2a2-36a0-489f-b938-d2af3b88619b","rayCluster":"counter-9qkzh","podIPs":["10.133.0.25","10.134.0.33"]}
{"level":"info","ts":"2026-06-11T10:19:05.097Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"a1777b73-8702-47b0-931e-a9f0fd9876be"}
{"level":"info","ts":"2026-06-11T10:19:05.097Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"a1777b73-8702-47b0-931e-a9f0fd9876be","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:05.117Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"514a7f6c-d64c-4dea-a3a1-22b8b557243e","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:19:05.129Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"514a7f6c-d64c-4dea-a3a1-22b8b557243e","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:19:05.129Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"514a7f6c-d64c-4dea-a3a1-22b8b557243e","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:06.921Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9970b81-56fc-4a7b-9e3e-fbef9f261536","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:06.921Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9970b81-56fc-4a7b-9e3e-fbef9f261536","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:06.921Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9970b81-56fc-4a7b-9e3e-fbef9f261536","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:06.921Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9970b81-56fc-4a7b-9e3e-fbef9f261536"}
{"level":"info","ts":"2026-06-11T10:19:06.921Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9970b81-56fc-4a7b-9e3e-fbef9f261536","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:19:07.097Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"a90ea996-9219-4faf-acd4-dc52573ad3cc","seconds":300}
{"level":"info","ts":"2026-06-11T10:19:09.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5cb8c23e-cb06-4d7e-b3e6-2ab81cb0f696","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:09.922Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5cb8c23e-cb06-4d7e-b3e6-2ab81cb0f696","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:09.922Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5cb8c23e-cb06-4d7e-b3e6-2ab81cb0f696","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:09.922Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5cb8c23e-cb06-4d7e-b3e6-2ab81cb0f696"}
{"level":"info","ts":"2026-06-11T10:19:09.922Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5cb8c23e-cb06-4d7e-b3e6-2ab81cb0f696","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247"}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247"}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:19:10.487Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:19:10.488Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:19:10.488Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"4012098f-17bb-4719-8457-8f71d85ef247","seconds":300}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5"}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5"}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:19:11.527Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"d4508093-106a-4ddf-b473-125cf86c5dc5","name":"counter-9qkzh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6cef09bc-ed9d-40d7-883b-6021e9c440d0","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6c2af1ad-685a-4c2a-a9fc-3e263f34c274","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b40dca33-2e71-487b-93db-ff5a45cc0761","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b40dca33-2e71-487b-93db-ff5a45cc0761","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b40dca33-2e71-487b-93db-ff5a45cc0761","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6c2af1ad-685a-4c2a-a9fc-3e263f34c274","pod":"counter-9qkzh-head-xhzgz","ip":"10.133.0.25"}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6c2af1ad-685a-4c2a-a9fc-3e263f34c274","pod":"counter-9qkzh-small-group-worker-szjpx","ip":"10.134.0.33"}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b40dca33-2e71-487b-93db-ff5a45cc0761"}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b40dca33-2e71-487b-93db-ff5a45cc0761","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:11.553Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6c2af1ad-685a-4c2a-a9fc-3e263f34c274","rayCluster":"counter-9qkzh","podIPs":["10.133.0.25","10.134.0.33"]}
{"level":"info","ts":"2026-06-11T10:19:11.577Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6cef09bc-ed9d-40d7-883b-6021e9c440d0","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:19:11.604Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6cef09bc-ed9d-40d7-883b-6021e9c440d0","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:19:11.604Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"6cef09bc-ed9d-40d7-883b-6021e9c440d0","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:12.923Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9510e3fc-88ac-42d9-8930-beec92ec23ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:12.923Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9510e3fc-88ac-42d9-8930-beec92ec23ad","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:12.923Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9510e3fc-88ac-42d9-8930-beec92ec23ad","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:12.923Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9510e3fc-88ac-42d9-8930-beec92ec23ad"}
{"level":"info","ts":"2026-06-11T10:19:12.923Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9510e3fc-88ac-42d9-8930-beec92ec23ad","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:13.373Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"be95fa2c-2d2d-4503-a422-1fcbfacec8d1","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:13.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"be95fa2c-2d2d-4503-a422-1fcbfacec8d1","pod":"counter-9qkzh-head-xhzgz","ip":"10.133.0.25"}
{"level":"info","ts":"2026-06-11T10:19:13.373Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"be95fa2c-2d2d-4503-a422-1fcbfacec8d1","pod":"counter-9qkzh-small-group-worker-szjpx","ip":"10.134.0.33"}
{"level":"info","ts":"2026-06-11T10:19:13.373Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"be95fa2c-2d2d-4503-a422-1fcbfacec8d1","rayCluster":"counter-9qkzh","podIPs":["10.133.0.25","10.134.0.33"]}
{"level":"info","ts":"2026-06-11T10:19:13.553Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789"}
{"level":"info","ts":"2026-06-11T10:19:13.553Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789"}
{"level":"info","ts":"2026-06-11T10:19:13.553Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","podName":"counter-9qkzh-small-group-worker-szjpx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:18:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:19:13.554Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"06da617e-9093-497b-a2e5-a4a89a489789","seconds":300}
{"level":"info","ts":"2026-06-11T10:19:15.924Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"28a6dd76-323d-4c58-a03d-0f71a0f0d62f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:15.924Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"28a6dd76-323d-4c58-a03d-0f71a0f0d62f","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:15.924Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"28a6dd76-323d-4c58-a03d-0f71a0f0d62f","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:15.924Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"28a6dd76-323d-4c58-a03d-0f71a0f0d62f"}
{"level":"info","ts":"2026-06-11T10:19:15.924Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"28a6dd76-323d-4c58-a03d-0f71a0f0d62f","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:16.585Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53785b98-9efe-461b-9535-a04ab8326d06","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:19:16.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53785b98-9efe-461b-9535-a04ab8326d06","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:19:16.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53785b98-9efe-461b-9535-a04ab8326d06","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:19:16.586Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"53785b98-9efe-461b-9535-a04ab8326d06","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:19:18.924Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5a6ceec7-b589-40e1-a24a-453ecc97970c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:18.924Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5a6ceec7-b589-40e1-a24a-453ecc97970c","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:18.924Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5a6ceec7-b589-40e1-a24a-453ecc97970c","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:18.924Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5a6ceec7-b589-40e1-a24a-453ecc97970c"}
{"level":"info","ts":"2026-06-11T10:19:18.924Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5a6ceec7-b589-40e1-a24a-453ecc97970c","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:21.925Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"99bdd332-4717-49db-b00f-5fa70c524d8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:21.925Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"99bdd332-4717-49db-b00f-5fa70c524d8f","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:21.925Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"99bdd332-4717-49db-b00f-5fa70c524d8f","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:21.926Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"99bdd332-4717-49db-b00f-5fa70c524d8f"}
{"level":"info","ts":"2026-06-11T10:19:21.926Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"99bdd332-4717-49db-b00f-5fa70c524d8f","RayCluster":"counter-9qkzh","State":""}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba"}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba"}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:19:22.518Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"9c87a4f3-cf2b-4c5f-bb79-b769a85647ba","name":"counter-9qkzh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"1c5bb972-58d3-4208-9a63-08a207dac62a","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d8017e40-6b20-480e-a3f9-ea4b385badea","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"1c5bb972-58d3-4208-9a63-08a207dac62a","pod":"counter-9qkzh-head-xhzgz","ip":"10.133.0.25"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"1c5bb972-58d3-4208-9a63-08a207dac62a","pod":"counter-9qkzh-small-group-worker-szjpx","ip":"10.134.0.33"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"1c5bb972-58d3-4208-9a63-08a207dac62a","rayCluster":"counter-9qkzh","podIPs":["10.133.0.25","10.134.0.33"]}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","head service name":"counter-9qkzh-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","head service URL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e"}
{"level":"info","ts":"2026-06-11T10:19:22.535Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","command":["if","!","ray","job","status","--address","http://counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","counter-dx8wq",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-dx8wq","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","--follow","counter-dx8wq"]}
{"level":"info","ts":"2026-06-11T10:19:22.545Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-11T10:19:22.545Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","SubmissionMode":"K8sJobMode","RayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.545Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:19:22.545Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2a22ecb8-6259-468d-8b96-effdc969820e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:19:22.552Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d8017e40-6b20-480e-a3f9-ea4b385badea","name":"counter-9qkzh-head"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","head service name":"counter-9qkzh-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","head service URL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","SubmissionMode":"K8sJobMode","RayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:19:22.560Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:19:22.569Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:19:22.569Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b"}
{"level":"error","ts":"2026-06-11T10:19:22.569Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"bb6b1eea-02da-463f-9b48-9e545c5af64b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.569Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"360902b3-f608-4689-8002-d285e0b01933","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.569Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"360902b3-f608-4689-8002-d285e0b01933","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.569Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"360902b3-f608-4689-8002-d285e0b01933","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.569Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"360902b3-f608-4689-8002-d285e0b01933"}
{"level":"error","ts":"2026-06-11T10:19:22.580Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"360902b3-f608-4689-8002-d285e0b01933","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.580Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"360902b3-f608-4689-8002-d285e0b01933"}
{"level":"error","ts":"2026-06-11T10:19:22.580Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"360902b3-f608-4689-8002-d285e0b01933","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3ecd092c-58eb-4e7a-ad1a-367ca2cdf215","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3ecd092c-58eb-4e7a-ad1a-367ca2cdf215","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3ecd092c-58eb-4e7a-ad1a-367ca2cdf215","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.580Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3ecd092c-58eb-4e7a-ad1a-367ca2cdf215"}
{"level":"info","ts":"2026-06-11T10:19:22.581Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d8017e40-6b20-480e-a3f9-ea4b385badea","name":"counter-9qkzh-workers"}
{"level":"info","ts":"2026-06-11T10:19:22.581Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"d8017e40-6b20-480e-a3f9-ea4b385badea","cluster":"counter-9qkzh"}
{"level":"error","ts":"2026-06-11T10:19:22.627Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3ecd092c-58eb-4e7a-ad1a-367ca2cdf215","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.627Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3ecd092c-58eb-4e7a-ad1a-367ca2cdf215"}
{"level":"error","ts":"2026-06-11T10:19:22.627Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3ecd092c-58eb-4e7a-ad1a-367ca2cdf215","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.627Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0626085c-b529-4031-b4a3-98c7fa3edf51","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.627Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0626085c-b529-4031-b4a3-98c7fa3edf51","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.627Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0626085c-b529-4031-b4a3-98c7fa3edf51","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.627Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0626085c-b529-4031-b4a3-98c7fa3edf51"}
{"level":"error","ts":"2026-06-11T10:19:22.633Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0626085c-b529-4031-b4a3-98c7fa3edf51","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.633Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0626085c-b529-4031-b4a3-98c7fa3edf51"}
{"level":"error","ts":"2026-06-11T10:19:22.633Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0626085c-b529-4031-b4a3-98c7fa3edf51","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.648Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2e072aab-00b7-4e65-8ae7-a05b3699fdfe","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.648Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2e072aab-00b7-4e65-8ae7-a05b3699fdfe","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.648Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2e072aab-00b7-4e65-8ae7-a05b3699fdfe","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.648Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2e072aab-00b7-4e65-8ae7-a05b3699fdfe"}
{"level":"error","ts":"2026-06-11T10:19:22.653Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2e072aab-00b7-4e65-8ae7-a05b3699fdfe","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.653Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2e072aab-00b7-4e65-8ae7-a05b3699fdfe"}
{"level":"error","ts":"2026-06-11T10:19:22.653Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2e072aab-00b7-4e65-8ae7-a05b3699fdfe","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.734Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"64142bc4-2559-4265-b69f-fd5c58b78160","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.734Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"64142bc4-2559-4265-b69f-fd5c58b78160","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.734Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"64142bc4-2559-4265-b69f-fd5c58b78160","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.734Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"64142bc4-2559-4265-b69f-fd5c58b78160"}
{"level":"error","ts":"2026-06-11T10:19:22.739Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"64142bc4-2559-4265-b69f-fd5c58b78160","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.739Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"64142bc4-2559-4265-b69f-fd5c58b78160"}
{"level":"error","ts":"2026-06-11T10:19:22.739Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"64142bc4-2559-4265-b69f-fd5c58b78160","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.900Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0e676658-9778-4057-ba50-38c48f4cde4e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:22.900Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0e676658-9778-4057-ba50-38c48f4cde4e","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:22.900Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0e676658-9778-4057-ba50-38c48f4cde4e","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:22.900Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0e676658-9778-4057-ba50-38c48f4cde4e"}
{"level":"error","ts":"2026-06-11T10:19:22.906Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0e676658-9778-4057-ba50-38c48f4cde4e","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:22.906Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0e676658-9778-4057-ba50-38c48f4cde4e"}
{"level":"error","ts":"2026-06-11T10:19:22.906Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"0e676658-9778-4057-ba50-38c48f4cde4e","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:23.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b244116e-21c6-4a65-8462-3a15b10d77c1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:23.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b244116e-21c6-4a65-8462-3a15b10d77c1","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:23.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b244116e-21c6-4a65-8462-3a15b10d77c1","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:23.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b244116e-21c6-4a65-8462-3a15b10d77c1"}
{"level":"error","ts":"2026-06-11T10:19:23.232Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b244116e-21c6-4a65-8462-3a15b10d77c1","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:23.232Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b244116e-21c6-4a65-8462-3a15b10d77c1"}
{"level":"error","ts":"2026-06-11T10:19:23.232Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"b244116e-21c6-4a65-8462-3a15b10d77c1","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:23.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4354f50a-99a7-42ec-b1de-7ac10aa5fd63","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:23.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4354f50a-99a7-42ec-b1de-7ac10aa5fd63","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:23.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4354f50a-99a7-42ec-b1de-7ac10aa5fd63","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:23.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4354f50a-99a7-42ec-b1de-7ac10aa5fd63"}
{"level":"error","ts":"2026-06-11T10:19:23.589Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4354f50a-99a7-42ec-b1de-7ac10aa5fd63","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:23.589Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4354f50a-99a7-42ec-b1de-7ac10aa5fd63"}
{"level":"error","ts":"2026-06-11T10:19:23.589Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4354f50a-99a7-42ec-b1de-7ac10aa5fd63","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:23.873Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5509feec-1474-4a98-9f26-566ec83908db","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:23.873Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5509feec-1474-4a98-9f26-566ec83908db","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:23.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5509feec-1474-4a98-9f26-566ec83908db","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:23.873Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5509feec-1474-4a98-9f26-566ec83908db"}
{"level":"error","ts":"2026-06-11T10:19:23.881Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5509feec-1474-4a98-9f26-566ec83908db","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:23.881Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5509feec-1474-4a98-9f26-566ec83908db"}
{"level":"error","ts":"2026-06-11T10:19:23.881Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"5509feec-1474-4a98-9f26-566ec83908db","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6"}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6"}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","1 head service found":"counter-9qkzh-head-svc"}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","Found 1 head Pod":"counter-9qkzh-head-xhzgz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","head Pod":"counter-9qkzh-head-xhzgz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-9qkzh-head-xhzgz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","worker Pod":"counter-9qkzh-small-group-worker-szjpx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-9qkzh-small-group-worker-szjpx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:19:24.535Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"270a5bd7-bbb7-47eb-8fa4-553296ee81c6","seconds":300}
{"level":"info","ts":"2026-06-11T10:19:26.442Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3118c078-c34d-4283-8c43-f485612d39f7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:26.442Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3118c078-c34d-4283-8c43-f485612d39f7","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:26.442Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3118c078-c34d-4283-8c43-f485612d39f7","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:26.442Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3118c078-c34d-4283-8c43-f485612d39f7"}
{"level":"error","ts":"2026-06-11T10:19:26.448Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3118c078-c34d-4283-8c43-f485612d39f7","JobId":"counter-dx8wq","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:26.448Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3118c078-c34d-4283-8c43-f485612d39f7"}
{"level":"error","ts":"2026-06-11T10:19:26.448Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"3118c078-c34d-4283-8c43-f485612d39f7","error":"Job counter-dx8wq does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:31.569Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"824f0518-0af7-44f0-9b82-65a945cb3c6c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:31.569Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"824f0518-0af7-44f0-9b82-65a945cb3c6c","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:31.569Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"824f0518-0af7-44f0-9b82-65a945cb3c6c","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:31.569Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"824f0518-0af7-44f0-9b82-65a945cb3c6c"}
{"level":"info","ts":"2026-06-11T10:19:31.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"824f0518-0af7-44f0-9b82-65a945cb3c6c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:31.576Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"824f0518-0af7-44f0-9b82-65a945cb3c6c","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:19:31.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"82eea590-db9d-4cf6-ae7d-317766d26d2b","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:31.594Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"82eea590-db9d-4cf6-ae7d-317766d26d2b","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:31.594Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"82eea590-db9d-4cf6-ae7d-317766d26d2b","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:31.594Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"82eea590-db9d-4cf6-ae7d-317766d26d2b"}
{"level":"info","ts":"2026-06-11T10:19:31.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"82eea590-db9d-4cf6-ae7d-317766d26d2b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:34.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"e4942bee-fda1-4491-8db7-5f5bdd39d460","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:34.593Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"e4942bee-fda1-4491-8db7-5f5bdd39d460","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:34.593Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"e4942bee-fda1-4491-8db7-5f5bdd39d460","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:34.593Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"e4942bee-fda1-4491-8db7-5f5bdd39d460"}
{"level":"info","ts":"2026-06-11T10:19:34.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"e4942bee-fda1-4491-8db7-5f5bdd39d460","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:34.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"e4942bee-fda1-4491-8db7-5f5bdd39d460","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:19:34.615Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11286761-d034-4bd6-8974-6c6f0ba4b159","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:34.615Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11286761-d034-4bd6-8974-6c6f0ba4b159","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:34.615Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11286761-d034-4bd6-8974-6c6f0ba4b159","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:34.615Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11286761-d034-4bd6-8974-6c6f0ba4b159"}
{"level":"info","ts":"2026-06-11T10:19:34.618Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"11286761-d034-4bd6-8974-6c6f0ba4b159","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:37.614Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"75a0f786-8909-4623-867c-08a8885abf37","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:37.614Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"75a0f786-8909-4623-867c-08a8885abf37","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:37.614Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"75a0f786-8909-4623-867c-08a8885abf37","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:37.614Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"75a0f786-8909-4623-867c-08a8885abf37"}
{"level":"info","ts":"2026-06-11T10:19:37.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"75a0f786-8909-4623-867c-08a8885abf37","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:40.615Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7260fb51-b1c0-4b14-8db7-6eb14a2278f4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:40.615Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7260fb51-b1c0-4b14-8db7-6eb14a2278f4","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:40.615Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7260fb51-b1c0-4b14-8db7-6eb14a2278f4","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:40.615Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7260fb51-b1c0-4b14-8db7-6eb14a2278f4"}
{"level":"info","ts":"2026-06-11T10:19:40.619Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"7260fb51-b1c0-4b14-8db7-6eb14a2278f4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:40.619Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9614565-e79a-48d7-98c9-e64f2b6a55c0","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:40.619Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9614565-e79a-48d7-98c9-e64f2b6a55c0","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:40.619Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9614565-e79a-48d7-98c9-e64f2b6a55c0","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:40.619Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9614565-e79a-48d7-98c9-e64f2b6a55c0"}
{"level":"info","ts":"2026-06-11T10:19:40.621Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f9614565-e79a-48d7-98c9-e64f2b6a55c0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:41.622Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2f172db2-8628-4a80-abeb-f38fa9cdafda","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:41.622Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2f172db2-8628-4a80-abeb-f38fa9cdafda","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.622Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2f172db2-8628-4a80-abeb-f38fa9cdafda","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.622Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2f172db2-8628-4a80-abeb-f38fa9cdafda"}
{"level":"info","ts":"2026-06-11T10:19:41.625Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"2f172db2-8628-4a80-abeb-f38fa9cdafda","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:41.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c0b3a8a1-dc20-4305-88d1-379bf6d4dd52","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:41.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c0b3a8a1-dc20-4305-88d1-379bf6d4dd52","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c0b3a8a1-dc20-4305-88d1-379bf6d4dd52","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c0b3a8a1-dc20-4305-88d1-379bf6d4dd52"}
{"level":"info","ts":"2026-06-11T10:19:41.645Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c0b3a8a1-dc20-4305-88d1-379bf6d4dd52","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:19:30Z","endTime":"2026-06-11T10:19:34Z"},"jobId":"counter-dx8wq","rayClusterName":"counter-9qkzh","dashboardURL":"counter-9qkzh-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-11T10:18:42Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:19:22Z","stateTransitionTimes":{"ready":"2026-06-11T10:19:22Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.25","serviceIP":"10.133.0.25","podName":"counter-9qkzh-head-xhzgz","serviceName":"counter-9qkzh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:19:05Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:19:22Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:18:42Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:18:43Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:19:41.645Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"c0b3a8a1-dc20-4305-88d1-379bf6d4dd52","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-11T10:19:41.661Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4d1e2c53-7063-4e46-b520-5712348bcec7","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:41.662Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4d1e2c53-7063-4e46-b520-5712348bcec7","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:19:41 +0000 UTC","Now":"2026-06-11T10:19:41.662Z","ShutdownTime":"2026-06-11T10:19:41.000Z"}
{"level":"info","ts":"2026-06-11T10:19:41.662Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4d1e2c53-7063-4e46-b520-5712348bcec7","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:19:41.678Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4d1e2c53-7063-4e46-b520-5712348bcec7","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.678Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4d1e2c53-7063-4e46-b520-5712348bcec7","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:19:41.678Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4d1e2c53-7063-4e46-b520-5712348bcec7","RayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.678Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"f2246202-a959-4b17-8632-0beef86e8229"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"reconcileID":"6b96eb6e-45a1-4c11-a014-7e001d800555"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"83ceda45-9f71-47ed-a347-23b1c52e007b","rayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"83ceda45-9f71-47ed-a347-23b1c52e007b","pod":"counter-9qkzh-small-group-worker-szjpx","ip":"10.134.0.33"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"195cfd25-147a-4d17-b02f-293f6baa33b4","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"195cfd25-147a-4d17-b02f-293f6baa33b4","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:19:41 +0000 UTC","Now":"2026-06-11T10:19:41.679Z","ShutdownTime":"2026-06-11T10:19:41.000Z"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"195cfd25-147a-4d17-b02f-293f6baa33b4","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"195cfd25-147a-4d17-b02f-293f6baa33b4","RayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"195cfd25-147a-4d17-b02f-293f6baa33b4","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"195cfd25-147a-4d17-b02f-293f6baa33b4","RayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"83ceda45-9f71-47ed-a347-23b1c52e007b","pod":"counter-9qkzh-head-xhzgz","ip":"10.133.0.25"}
{"level":"info","ts":"2026-06-11T10:19:41.679Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"83ceda45-9f71-47ed-a347-23b1c52e007b","rayCluster":"counter-9qkzh","podIPs":["10.134.0.33","10.133.0.25"]}
{"level":"info","ts":"2026-06-11T10:19:41.686Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","name":"kuberay-gateway-access","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:19:41.691Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","name":"test-ns-c6srn-counter-9qkzh","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:19:41.697Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","configMap":"kube-rbac-proxy-config-counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.702Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","serviceAccount":"counter-9qkzh-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:19:41.709Z","logger":"raycluster-resource","msg":"default","name":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.709Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-9qkzh","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"7c663ca4-bf7f-469e-8a69-cce8d1dfb224","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73","cluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"cc9d4e4b-21e0-4da2-8d4a-57ed439c1709"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"36bf8e0b-4263-48ca-b6d2-01a4efe0f514","namespace":"test-ns-c6srn","clusterName":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4dfa6907-9c5f-4bc6-8be9-90fcf2a311fa","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4dfa6907-9c5f-4bc6-8be9-90fcf2a311fa","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:19:41 +0000 UTC","Now":"2026-06-11T10:19:41.720Z","ShutdownTime":"2026-06-11T10:19:41.000Z"}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4dfa6907-9c5f-4bc6-8be9-90fcf2a311fa","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4dfa6907-9c5f-4bc6-8be9-90fcf2a311fa","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4dfa6907-9c5f-4bc6-8be9-90fcf2a311fa","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:19:41.720Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"4dfa6907-9c5f-4bc6-8be9-90fcf2a311fa","RayCluster":"counter-9qkzh"}
{"level":"error","ts":"2026-06-11T10:19:41.724Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73","error":"rayclusters.ray.io \"counter-9qkzh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:41.724Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73"}
{"level":"error","ts":"2026-06-11T10:19:41.724Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"410ade0e-e6eb-4a96-be94-13f7824b3a73","error":"rayclusters.ray.io \"counter-9qkzh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:41.724Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2fc6de88-2df2-4d5a-b59d-43527c764eca","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.724Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2fc6de88-2df2-4d5a-b59d-43527c764eca"}
{"level":"info","ts":"2026-06-11T10:19:41.724Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2fc6de88-2df2-4d5a-b59d-43527c764eca","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:19:41.724Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"2fc6de88-2df2-4d5a-b59d-43527c764eca","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:19:41.729Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"60f0dd3f-a215-4240-906f-4ff582e34f87","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.729Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"60f0dd3f-a215-4240-906f-4ff582e34f87"}
{"level":"info","ts":"2026-06-11T10:19:41.729Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"60f0dd3f-a215-4240-906f-4ff582e34f87","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:19:41.729Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"60f0dd3f-a215-4240-906f-4ff582e34f87","namespace":"test-ns-c6srn"}
{"level":"error","ts":"2026-06-11T10:19:41.731Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"36bf8e0b-4263-48ca-b6d2-01a4efe0f514","issuer":"ray-selfsigned-issuer-counter-9qkzh","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-9qkzh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:19:41.734Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"36bf8e0b-4263-48ca-b6d2-01a4efe0f514","certificate":"ray-head-cert-counter-9qkzh","error":"certificates.cert-manager.io \"ray-head-cert-counter-9qkzh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:41.739Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4214f33c-6b38-494b-8eb2-81e7bf1891b8","namespacedName":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:41.739Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4214f33c-6b38-494b-8eb2-81e7bf1891b8"}
{"level":"info","ts":"2026-06-11T10:19:41.739Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4214f33c-6b38-494b-8eb2-81e7bf1891b8","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:19:41.739Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"4214f33c-6b38-494b-8eb2-81e7bf1891b8","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:19:41.742Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"968bd1dc-1d4d-48d5-a466-5bd057dd390e"}
{"level":"info","ts":"2026-06-11T10:19:41.742Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"bd265ee3-c713-4088-89d7-129ae350dbb4"}
{"level":"info","ts":"2026-06-11T10:19:41.768Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"36bf8e0b-4263-48ca-b6d2-01a4efe0f514","errorCount":2}
{"level":"info","ts":"2026-06-11T10:19:41.768Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"36bf8e0b-4263-48ca-b6d2-01a4efe0f514"}
{"level":"error","ts":"2026-06-11T10:19:41.768Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"36bf8e0b-4263-48ca-b6d2-01a4efe0f514","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-9qkzh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:19:41.773Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"8628e5ac-4366-460a-9f67-e7c1acb453d2","namespace":"test-ns-c6srn","clusterName":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:41.773Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"counter-9qkzh","reconcileID":"8628e5ac-4366-460a-9f67-e7c1acb453d2","namespace":"test-ns-c6srn","clusterName":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:42.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9246cb22-4f3e-426c-8b99-53c39243e885","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:42.252Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9246cb22-4f3e-426c-8b99-53c39243e885","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:19:41 +0000 UTC","Now":"2026-06-11T10:19:42.252Z","ShutdownTime":"2026-06-11T10:19:41.000Z"}
{"level":"info","ts":"2026-06-11T10:19:42.252Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9246cb22-4f3e-426c-8b99-53c39243e885","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:19:42.252Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9246cb22-4f3e-426c-8b99-53c39243e885","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:42.252Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9246cb22-4f3e-426c-8b99-53c39243e885","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:19:42.252Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"9246cb22-4f3e-426c-8b99-53c39243e885","RayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:19:43.619Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f2af68a0-5774-47f7-9da9-c4edc5ca4d3c","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:19:43.619Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f2af68a0-5774-47f7-9da9-c4edc5ca4d3c","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:19:41 +0000 UTC","Now":"2026-06-11T10:19:43.619Z","ShutdownTime":"2026-06-11T10:19:41.000Z"}
{"level":"info","ts":"2026-06-11T10:19:43.619Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f2af68a0-5774-47f7-9da9-c4edc5ca4d3c","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:19:43.619Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f2af68a0-5774-47f7-9da9-c4edc5ca4d3c","RayCluster":{"name":"counter-9qkzh","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:19:43.619Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f2af68a0-5774-47f7-9da9-c4edc5ca4d3c","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:19:43.619Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"f2af68a0-5774-47f7-9da9-c4edc5ca4d3c","RayCluster":"counter-9qkzh"}
{"level":"info","ts":"2026-06-11T10:20:12.265Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"a82f71a4-f0cb-4a03-a031-8da3e23d157f","DeletionTimestamp":"2026-06-11 10:20:12 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:20:12.265Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"a82f71a4-f0cb-4a03-a031-8da3e23d157f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:20:12.279Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"24a86e3b-e6e3-4c7e-8f05-649f84b231f0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:12.279Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"24a86e3b-e6e3-4c7e-8f05-649f84b231f0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:20:12.290Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"24a86e3b-e6e3-4c7e-8f05-649f84b231f0"}
{"level":"info","ts":"2026-06-11T10:20:12.290Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"24a86e3b-e6e3-4c7e-8f05-649f84b231f0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:20:12.290Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"24a86e3b-e6e3-4c7e-8f05-649f84b231f0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:20:12.290Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"24a86e3b-e6e3-4c7e-8f05-649f84b231f0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:20:12.302Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"10c7c4f8-349e-428a-bea7-ce35c00c2bc9"}
{"level":"info","ts":"2026-06-11T10:20:12.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"426a8f9f-41cc-4cad-b94d-967ce922b2b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:12.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"426a8f9f-41cc-4cad-b94d-967ce922b2b9","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.302Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"426a8f9f-41cc-4cad-b94d-967ce922b2b9","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.302Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"426a8f9f-41cc-4cad-b94d-967ce922b2b9","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.307Z","logger":"raycluster-resource","msg":"default","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.308Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-88qlb","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2dbf606a-bdf9-43b3-bfb3-661fc635e047","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2dbf606a-bdf9-43b3-bfb3-661fc635e047","mode":"IntegratedOAuth","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2dbf606a-bdf9-43b3-bfb3-661fc635e047","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2dbf606a-bdf9-43b3-bfb3-661fc635e047","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.325Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"b9872d76-e053-48fd-8df5-4d11e9b7b996","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"426a8f9f-41cc-4cad-b94d-967ce922b2b9","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.427Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","name":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"raycluster-resource","msg":"default","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"426a8f9f-41cc-4cad-b94d-967ce922b2b9"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"426a8f9f-41cc-4cad-b94d-967ce922b2b9","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"b6564185-521d-4bb9-aff2-97e0bb940c7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"b6564185-521d-4bb9-aff2-97e0bb940c7d","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","name":"fail-88qlb","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"b6564185-521d-4bb9-aff2-97e0bb940c7d","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"b6564185-521d-4bb9-aff2-97e0bb940c7d"}
{"level":"info","ts":"2026-06-11T10:20:12.428Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"b6564185-521d-4bb9-aff2-97e0bb940c7d","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:12.434Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"b9872d76-e053-48fd-8df5-4d11e9b7b996","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.434Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2bfc90f7-1cbd-4b18-8d4c-f723ab3d8fac","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.440Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be"}
{"level":"error","ts":"2026-06-11T10:20:12.440Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4c9a79a-446f-4784-8b7c-1d04997439be","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.441Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8"}
{"level":"info","ts":"2026-06-11T10:20:12.441Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:20:12.441Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.527Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-88qlb","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:20:12.527Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235"}
{"level":"info","ts":"2026-06-11T10:20:12.527Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235"}
{"level":"info","ts":"2026-06-11T10:20:12.527Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:12.527Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:12.527Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.527Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:12.537Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"bfb1866f-1db1-49fa-8eab-f8289655243f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"bfb1866f-1db1-49fa-8eab-f8289655243f","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"bfb1866f-1db1-49fa-8eab-f8289655243f","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235"}
{"level":"error","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"90cf1b66-7ad8-43d2-b8cd-daff02744235","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f"}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f"}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"bfb1866f-1db1-49fa-8eab-f8289655243f"}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"bfb1866f-1db1-49fa-8eab-f8289655243f","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8"}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f"}
{"level":"error","ts":"2026-06-11T10:20:12.628Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4f487a78-894b-4d48-8f3b-f800b871bc6f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.632Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2bfc90f7-1cbd-4b18-8d4c-f723ab3d8fac","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc"}
{"level":"error","ts":"2026-06-11T10:20:12.638Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3318d246-6483-4d80-abc7-1e76082c4fcc","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"raycluster-resource","msg":"default","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.638Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-88qlb","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:20:12.639Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8"}
{"level":"info","ts":"2026-06-11T10:20:12.639Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","rayCluster":"fail-88qlb","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:20:12.639Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-88qlb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.639Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8"}
{"level":"error","ts":"2026-06-11T10:20:12.639Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"403e1abf-3739-44ff-8961-0eb770c133b8","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-88qlb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.639Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"542b5c48-09b7-4be4-bee7-aa1d1238ca75","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.639Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"542b5c48-09b7-4be4-bee7-aa1d1238ca75"}
{"level":"error","ts":"2026-06-11T10:20:12.641Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2dbf606a-bdf9-43b3-bfb3-661fc635e047","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.641Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2dbf606a-bdf9-43b3-bfb3-661fc635e047"}
{"level":"error","ts":"2026-06-11T10:20:12.641Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2dbf606a-bdf9-43b3-bfb3-661fc635e047","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.641Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.641Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","mode":"IntegratedOAuth","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.641Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.641Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.644Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2bfc90f7-1cbd-4b18-8d4c-f723ab3d8fac","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:12.644Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2bfc90f7-1cbd-4b18-8d4c-f723ab3d8fac","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.644Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a55af5c4-7474-4ab4-be23-4261fa3c96a6","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.644Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"d7956a51-638f-478f-9669-4c87e8bd9eda","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.644Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"d7956a51-638f-478f-9669-4c87e8bd9eda"}
{"level":"info","ts":"2026-06-11T10:20:12.647Z","logger":"raycluster-resource","msg":"default","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.647Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-88qlb","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:20:12.658Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a55af5c4-7474-4ab4-be23-4261fa3c96a6","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336"}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336"}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336"}
{"level":"error","ts":"2026-06-11T10:20:12.727Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3e21b6dd-cb41-4f67-b5ac-cf72b6e9c336","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.732Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a55af5c4-7474-4ab4-be23-4261fa3c96a6","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:12.732Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a55af5c4-7474-4ab4-be23-4261fa3c96a6","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"5cfec71d-eb2b-497d-84a4-51fc1e7297c0","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"130669d5-1aac-45f9-85b5-99217eb41ce9","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1eddc30a-6f00-470b-83ca-b63c26d590ba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1eddc30a-6f00-470b-83ca-b63c26d590ba","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1eddc30a-6f00-470b-83ca-b63c26d590ba","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"130669d5-1aac-45f9-85b5-99217eb41ce9"}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1eddc30a-6f00-470b-83ca-b63c26d590ba"}
{"level":"info","ts":"2026-06-11T10:20:12.738Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1eddc30a-6f00-470b-83ca-b63c26d590ba","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:12.744Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","name":"fail-88qlb-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:20:12.749Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"5cfec71d-eb2b-497d-84a4-51fc1e7297c0","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:12.749Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","name":"kuberay-gateway-access","namespace":"test-ns-c6srn","operation":"created"}
{"level":"info","ts":"2026-06-11T10:20:12.759Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","name":"test-ns-c6srn-fail-88qlb","operation":"created"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","name":"kube-rbac-proxy-config-fail-88qlb","operation":"created"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"5cfec71d-eb2b-497d-84a4-51fc1e7297c0","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"5cfec71d-eb2b-497d-84a4-51fc1e7297c0","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.828Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee"}
{"level":"error","ts":"2026-06-11T10:20:12.828Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d21103ee-b480-47e4-8207-b5e3e92279ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:20:12.835Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.835Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847"}
{"level":"error","ts":"2026-06-11T10:20:12.835Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"9ba1cb94-e81b-4b8e-8f2f-b77f9a9c3847","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.835Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4492def2-a3cc-481e-af37-56da05f9ab4f","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.835Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4492def2-a3cc-481e-af37-56da05f9ab4f","mode":"IntegratedOAuth","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.835Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4492def2-a3cc-481e-af37-56da05f9ab4f","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.848Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4492def2-a3cc-481e-af37-56da05f9ab4f","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.848Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"cf412646-8b67-454f-81b9-cbfb44929096","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"cf412646-8b67-454f-81b9-cbfb44929096","mode":"IntegratedOAuth","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"cf412646-8b67-454f-81b9-cbfb44929096","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a84fcd81-a438-4f69-85ac-603257db3e83","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"0eea9761-7686-4bf9-9528-2a596a8e7efd","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"0eea9761-7686-4bf9-9528-2a596a8e7efd"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1ec12b4f-c299-4937-8e1a-c2687a1ab628","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1ec12b4f-c299-4937-8e1a-c2687a1ab628","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1ec12b4f-c299-4937-8e1a-c2687a1ab628","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1ec12b4f-c299-4937-8e1a-c2687a1ab628"}
{"level":"info","ts":"2026-06-11T10:20:12.849Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1ec12b4f-c299-4937-8e1a-c2687a1ab628","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:12.863Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a84fcd81-a438-4f69-85ac-603257db3e83","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:12.931Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a84fcd81-a438-4f69-85ac-603257db3e83","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:12.931Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a84fcd81-a438-4f69-85ac-603257db3e83","cluster":"fail-88qlb"}
{"level":"error","ts":"2026-06-11T10:20:12.934Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"cf412646-8b67-454f-81b9-cbfb44929096","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.934Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"cf412646-8b67-454f-81b9-cbfb44929096"}
{"level":"error","ts":"2026-06-11T10:20:12.934Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"cf412646-8b67-454f-81b9-cbfb44929096","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.940Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"18982a9d-60c1-4940-a7c4-8691eb74550d","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:12.940Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"18982a9d-60c1-4940-a7c4-8691eb74550d","mode":"IntegratedOAuth","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.940Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"18982a9d-60c1-4940-a7c4-8691eb74550d","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.950Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"18982a9d-60c1-4940-a7c4-8691eb74550d","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220"}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220"}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220"}
{"level":"error","ts":"2026-06-11T10:20:12.988Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7a49a469-0bc0-42ec-bb64-ae20157ee220","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c"}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c"}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","error":"head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"error","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found"}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-88qlb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c"}
{"level":"error","ts":"2026-06-11T10:20:13.309Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"467356ca-2194-4f2b-a594-93fd457ca96c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-88qlb not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","mode":"IntegratedOAuth","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","cluster":"fail-88qlb","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","cluster":"fail-88qlb","authType":"IntegratedOAuth","serviceAccount":"fail-88qlb-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-88qlb","ray.io/group":"headgroup","ray.io/identifier":"fail-88qlb-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:20:13.949Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:20:13.950Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:20:13.950Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:20:13.950Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","enabled":true}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","name":"fail-88qlb-head-k2rbf"}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52"}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","nodeType":"worker","rayStartParams":{"address":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","rayStartCmd":"ray start  --address=fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:20:13.972Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","enabled":true}
{"level":"info","ts":"2026-06-11T10:20:13.989Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","name":"fail-88qlb-small-group-worker-p7c6v"}
{"level":"info","ts":"2026-06-11T10:20:13.989Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:13.989Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:13.989Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-88qlb-head-svc"},"newHead":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"}}
{"level":"info","ts":"2026-06-11T10:20:13.989Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3918f4ab-5828-4265-b345-208f2e03ce52","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:14.005Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020"}
{"level":"info","ts":"2026-06-11T10:20:14.005Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020"}
{"level":"info","ts":"2026-06-11T10:20:14.005Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-88qlb-head-svc"},"newHead":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"}}
{"level":"info","ts":"2026-06-11T10:20:14.006Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:14.007Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"ec049ad2-8dc6-46f8-a1ee-7c83748d68cf","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.007Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"ec049ad2-8dc6-46f8-a1ee-7c83748d68cf"}
{"level":"info","ts":"2026-06-11T10:20:14.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"dc74f192-4c7c-425e-a31a-ddcbc69d38ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:14.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"dc74f192-4c7c-425e-a31a-ddcbc69d38ff","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"dc74f192-4c7c-425e-a31a-ddcbc69d38ff","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:14.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"dc74f192-4c7c-425e-a31a-ddcbc69d38ff"}
{"level":"info","ts":"2026-06-11T10:20:14.008Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"dc74f192-4c7c-425e-a31a-ddcbc69d38ff","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:14.008Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"07d4aa91-b125-478d-ac43-93ae43146595","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.014Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","name":"fail-88qlb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"fail-88qlb"}}
{"level":"info","ts":"2026-06-11T10:20:14.014Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020"}
{"level":"error","ts":"2026-06-11T10:20:14.014Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1bcdc83e-b5af-4dcc-8eee-9c65d09df020","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.027Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.028Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:20:14.028Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"65d1d3d8-34e3-4e9a-8f3c-2240e21cad74","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:14.033Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"07d4aa91-b125-478d-ac43-93ae43146595","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"d102297c-e95b-4ce5-b353-30a1b48202c7","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"d102297c-e95b-4ce5-b353-30a1b48202c7"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5e69c21f-8202-4568-9d3f-f8f8a6c9e054","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5e69c21f-8202-4568-9d3f-f8f8a6c9e054","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.041Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5e69c21f-8202-4568-9d3f-f8f8a6c9e054","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:14.042Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5e69c21f-8202-4568-9d3f-f8f8a6c9e054"}
{"level":"info","ts":"2026-06-11T10:20:14.042Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5e69c21f-8202-4568-9d3f-f8f8a6c9e054","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:14.045Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"07d4aa91-b125-478d-ac43-93ae43146595","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:14.045Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"07d4aa91-b125-478d-ac43-93ae43146595","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.045Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4e3210e9-1fe4-414a-ae05-23b3d6dd804f","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.048Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","name":"fail-88qlb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"fail-88qlb"}}
{"level":"info","ts":"2026-06-11T10:20:14.048Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741"}
{"level":"error","ts":"2026-06-11T10:20:14.048Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"bc6902a8-abb3-4f20-bbdd-80df03694741","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-88qlb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa"}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa"}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.053Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.054Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.054Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:14.054Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"d27bb3be-6236-4d64-b051-fb5f20ee46fa","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:14.062Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4e3210e9-1fe4-414a-ae05-23b3d6dd804f","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:14.132Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4e3210e9-1fe4-414a-ae05-23b3d6dd804f","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:14.132Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"4e3210e9-1fe4-414a-ae05-23b3d6dd804f","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:14.437Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4"}
{"level":"info","ts":"2026-06-11T10:20:14.437Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4"}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:14.438Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"2ec37107-282d-459e-a600-51d0269ee8e4","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:14.443Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"f5b93ce8-ecc1-4759-8b08-dbb8f6b045cf","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:14.773Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71"}
{"level":"info","ts":"2026-06-11T10:20:14.773Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71"}
{"level":"info","ts":"2026-06-11T10:20:14.773Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:14.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:14.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:14.774Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"07774c38-588b-4d37-aa33-54990ed54b71","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:15.280Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-c6srn"},"reconcileID":"e2286d39-733a-4ffd-9bfb-70d30f093225"}
{"level":"info","ts":"2026-06-11T10:20:15.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4151e271-baf4-46b0-a28c-e1ea4aadff59","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:15.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4151e271-baf4-46b0-a28c-e1ea4aadff59","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:15.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4151e271-baf4-46b0-a28c-e1ea4aadff59","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:15.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4151e271-baf4-46b0-a28c-e1ea4aadff59"}
{"level":"info","ts":"2026-06-11T10:20:15.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4151e271-baf4-46b0-a28c-e1ea4aadff59","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:15.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f"}
{"level":"info","ts":"2026-06-11T10:20:15.361Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f"}
{"level":"info","ts":"2026-06-11T10:20:15.361Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:15.361Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"newHead":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"}}
{"level":"info","ts":"2026-06-11T10:20:15.362Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e0803afa-f91a-464b-a921-1eac524eed8f","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"7c129f8a-899e-4d4d-9326-d238e9b73c57","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b384868-c53d-485d-912c-7ede81835da9","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"7c129f8a-899e-4d4d-9326-d238e9b73c57","pod":"fail-88qlb-head-k2rbf","ip":"10.133.0.26"}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"7c129f8a-899e-4d4d-9326-d238e9b73c57","pod":"fail-88qlb-small-group-worker-p7c6v","ip":"10.134.0.35"}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ef96fda4-f79b-44f0-9b72-224f57a85f5c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ef96fda4-f79b-44f0-9b72-224f57a85f5c","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ef96fda4-f79b-44f0-9b72-224f57a85f5c","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ef96fda4-f79b-44f0-9b72-224f57a85f5c"}
{"level":"info","ts":"2026-06-11T10:20:15.375Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ef96fda4-f79b-44f0-9b72-224f57a85f5c","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:15.388Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b384868-c53d-485d-912c-7ede81835da9","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:15.389Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"7c129f8a-899e-4d4d-9326-d238e9b73c57","certificate":"ray-head-cert-fail-88qlb","dnsNames":["fail-88qlb-head-svc","localhost","fail-88qlb-head-svc.test-ns-c6srn.svc","fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.26","10.134.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:20:15.401Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b384868-c53d-485d-912c-7ede81835da9","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:15.401Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b384868-c53d-485d-912c-7ede81835da9","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:15.402Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"7c129f8a-899e-4d4d-9326-d238e9b73c57","certificate":"ray-worker-cert-fail-88qlb","dnsNames":["fail-88qlb-worker-svc","localhost","fail-88qlb-small-group","fail-88qlb-worker-svc.test-ns-c6srn.svc","fail-88qlb-worker-svc.test-ns-c6srn.svc.cluster.local","*.fail-88qlb-worker-svc.test-ns-c6srn.svc","*.fail-88qlb-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.26","10.134.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:20:15.402Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"7c129f8a-899e-4d4d-9326-d238e9b73c57","rayCluster":"fail-88qlb","podIPs":["10.133.0.26","10.134.0.35"]}
{"level":"info","ts":"2026-06-11T10:20:16.586Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b702587e-3f31-4b8d-965d-ed8a975ee6be","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:20:16.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b702587e-3f31-4b8d-965d-ed8a975ee6be","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:20:16.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b702587e-3f31-4b8d-965d-ed8a975ee6be","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:20:16.586Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"b702587e-3f31-4b8d-965d-ed8a975ee6be","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:20:17.375Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f"}
{"level":"info","ts":"2026-06-11T10:20:17.375Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f"}
{"level":"info","ts":"2026-06-11T10:20:17.375Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:17.376Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"8475ca86-d3d8-4bfd-990e-d4baf43f792f","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:18.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ebe40ad1-88b5-4ac9-9efb-83880429331c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:18.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ebe40ad1-88b5-4ac9-9efb-83880429331c","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:18.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ebe40ad1-88b5-4ac9-9efb-83880429331c","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:18.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ebe40ad1-88b5-4ac9-9efb-83880429331c"}
{"level":"info","ts":"2026-06-11T10:20:18.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"ebe40ad1-88b5-4ac9-9efb-83880429331c","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:20.384Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b"}
{"level":"info","ts":"2026-06-11T10:20:20.384Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b"}
{"level":"info","ts":"2026-06-11T10:20:20.384Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","podName":"fail-88qlb-head-k2rbf","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:20.385Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"dc0c8058-4e33-418f-b971-74a7f40dee6b","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:21.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a0d32eba-7426-4d6d-aa0f-aedbde467dc2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:21.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a0d32eba-7426-4d6d-aa0f-aedbde467dc2","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:21.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a0d32eba-7426-4d6d-aa0f-aedbde467dc2","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:21.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a0d32eba-7426-4d6d-aa0f-aedbde467dc2"}
{"level":"info","ts":"2026-06-11T10:20:21.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a0d32eba-7426-4d6d-aa0f-aedbde467dc2","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe"}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe"}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","podName":"fail-88qlb-head-k2rbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","podName":"fail-88qlb-head-k2rbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:20:21.385Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"1d1b4c2b-b479-42a1-9bd0-985a4a6bebfe","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:21.397Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"8fc6d216-0616-4d3d-9de9-f97bc4d31126","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:21.397Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"93fab588-a918-4d2f-b91b-c2a4de7a3caf","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:21.397Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"8fc6d216-0616-4d3d-9de9-f97bc4d31126","pod":"fail-88qlb-head-k2rbf","ip":"10.133.0.26"}
{"level":"info","ts":"2026-06-11T10:20:21.397Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"8fc6d216-0616-4d3d-9de9-f97bc4d31126","pod":"fail-88qlb-small-group-worker-p7c6v","ip":"10.134.0.35"}
{"level":"info","ts":"2026-06-11T10:20:21.398Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"8fc6d216-0616-4d3d-9de9-f97bc4d31126","rayCluster":"fail-88qlb","podIPs":["10.133.0.26","10.134.0.35"]}
{"level":"info","ts":"2026-06-11T10:20:21.398Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"347a2bfc-110c-4dcc-a711-d938bb586d81","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:21.398Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"347a2bfc-110c-4dcc-a711-d938bb586d81","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:21.398Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"347a2bfc-110c-4dcc-a711-d938bb586d81","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:21.398Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"347a2bfc-110c-4dcc-a711-d938bb586d81"}
{"level":"info","ts":"2026-06-11T10:20:21.398Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"347a2bfc-110c-4dcc-a711-d938bb586d81","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:21.412Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"93fab588-a918-4d2f-b91b-c2a4de7a3caf","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:21.422Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"93fab588-a918-4d2f-b91b-c2a4de7a3caf","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:21.422Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"93fab588-a918-4d2f-b91b-c2a4de7a3caf","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:23.397Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf"}
{"level":"info","ts":"2026-06-11T10:20:23.397Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf"}
{"level":"info","ts":"2026-06-11T10:20:23.397Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:23.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:23.397Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","podName":"fail-88qlb-head-k2rbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","podName":"fail-88qlb-head-k2rbf","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:23.398Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"3703fe64-e479-4bac-b4bf-18b1486b1bcf","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:24.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6d196d5d-dd80-4524-9196-9d0f53f80617","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:24.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6d196d5d-dd80-4524-9196-9d0f53f80617","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:24.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6d196d5d-dd80-4524-9196-9d0f53f80617","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:24.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6d196d5d-dd80-4524-9196-9d0f53f80617"}
{"level":"info","ts":"2026-06-11T10:20:24.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6d196d5d-dd80-4524-9196-9d0f53f80617","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:27.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fc044f83-a4fc-46af-b9da-15991d278c03","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:27.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fc044f83-a4fc-46af-b9da-15991d278c03","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:27.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fc044f83-a4fc-46af-b9da-15991d278c03","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:27.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fc044f83-a4fc-46af-b9da-15991d278c03"}
{"level":"info","ts":"2026-06-11T10:20:27.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fc044f83-a4fc-46af-b9da-15991d278c03","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:30.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"789336cc-1298-4080-a88a-d08212fc8f1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:30.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"789336cc-1298-4080-a88a-d08212fc8f1d","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:30.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"789336cc-1298-4080-a88a-d08212fc8f1d","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:30.308Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"789336cc-1298-4080-a88a-d08212fc8f1d"}
{"level":"info","ts":"2026-06-11T10:20:30.308Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"789336cc-1298-4080-a88a-d08212fc8f1d","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:20:32.419Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"275f81d7-5cc1-4c5d-92d9-a000461090e1","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:32.432Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"01a789a3-abf6-4195-9752-0b5b17e78c0a","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:32.432Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"f992f122-b3e1-4204-a2ac-55ed46163e52","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"f992f122-b3e1-4204-a2ac-55ed46163e52","pod":"fail-88qlb-head-k2rbf","ip":"10.133.0.26"}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"f992f122-b3e1-4204-a2ac-55ed46163e52","pod":"fail-88qlb-small-group-worker-p7c6v","ip":"10.134.0.35"}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"c1ea596f-9d26-43ca-9d4c-6571e1a534e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"f992f122-b3e1-4204-a2ac-55ed46163e52","rayCluster":"fail-88qlb","podIPs":["10.133.0.26","10.134.0.35"]}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"c1ea596f-9d26-43ca-9d4c-6571e1a534e8","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"c1ea596f-9d26-43ca-9d4c-6571e1a534e8","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"c1ea596f-9d26-43ca-9d4c-6571e1a534e8"}
{"level":"info","ts":"2026-06-11T10:20:32.433Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"c1ea596f-9d26-43ca-9d4c-6571e1a534e8","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:32.446Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"01a789a3-abf6-4195-9752-0b5b17e78c0a","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:32.466Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"01a789a3-abf6-4195-9752-0b5b17e78c0a","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:32.466Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"01a789a3-abf6-4195-9752-0b5b17e78c0a","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:33.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f828aefc-9875-4607-add3-cc9042c5879e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:33.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f828aefc-9875-4607-add3-cc9042c5879e","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:33.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f828aefc-9875-4607-add3-cc9042c5879e","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:33.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f828aefc-9875-4607-add3-cc9042c5879e"}
{"level":"info","ts":"2026-06-11T10:20:33.309Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f828aefc-9875-4607-add3-cc9042c5879e","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:34.434Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4bb89bee-5d57-477c-89b9-45efee3d6f50","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:36.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"681061da-70cb-42a2-b914-70f62536b074","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:36.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"681061da-70cb-42a2-b914-70f62536b074","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:36.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"681061da-70cb-42a2-b914-70f62536b074","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:36.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"681061da-70cb-42a2-b914-70f62536b074"}
{"level":"info","ts":"2026-06-11T10:20:36.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"681061da-70cb-42a2-b914-70f62536b074","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:37.885Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"b4267cd1-3fa8-4965-9c1f-465ff8eb438c","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53"}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53"}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:38.891Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:20:38.891Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:20:38.891Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:20:38.891Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"23336765-7ba1-4370-ac3e-7dc7b31a7d53","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"c0a65e96-4418-4dbd-ba86-00b8d3a52823","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2fdc0298-f06c-4cd5-b1bd-ce0fbd34e555","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"c0a65e96-4418-4dbd-ba86-00b8d3a52823","pod":"fail-88qlb-small-group-worker-p7c6v","ip":"10.134.0.35"}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"c0a65e96-4418-4dbd-ba86-00b8d3a52823","pod":"fail-88qlb-head-k2rbf","ip":"10.133.0.26"}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"aa9132b5-1687-4767-b122-91df489108a4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"aa9132b5-1687-4767-b122-91df489108a4","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"c0a65e96-4418-4dbd-ba86-00b8d3a52823","rayCluster":"fail-88qlb","podIPs":["10.134.0.35","10.133.0.26"]}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"aa9132b5-1687-4767-b122-91df489108a4","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"aa9132b5-1687-4767-b122-91df489108a4"}
{"level":"info","ts":"2026-06-11T10:20:38.904Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"aa9132b5-1687-4767-b122-91df489108a4","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:38.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2fdc0298-f06c-4cd5-b1bd-ce0fbd34e555","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:38.933Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2fdc0298-f06c-4cd5-b1bd-ce0fbd34e555","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:38.933Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"2fdc0298-f06c-4cd5-b1bd-ce0fbd34e555","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:39.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"be8bd4c6-85a4-4eb2-8dbd-277f9391a02c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:39.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"be8bd4c6-85a4-4eb2-8dbd-277f9391a02c","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:39.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"be8bd4c6-85a4-4eb2-8dbd-277f9391a02c","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:39.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"be8bd4c6-85a4-4eb2-8dbd-277f9391a02c"}
{"level":"info","ts":"2026-06-11T10:20:39.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"be8bd4c6-85a4-4eb2-8dbd-277f9391a02c","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce"}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce"}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","podName":"fail-88qlb-small-group-worker-p7c6v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:20:14Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:40.904Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"4207c3b1-ccdc-4e17-9b87-fa86d8e8c1ce","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:42.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"65ba44aa-5659-4d72-98f3-d9f954c79488","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:42.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"65ba44aa-5659-4d72-98f3-d9f954c79488","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:42.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"65ba44aa-5659-4d72-98f3-d9f954c79488","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:42.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"65ba44aa-5659-4d72-98f3-d9f954c79488"}
{"level":"info","ts":"2026-06-11T10:20:42.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"65ba44aa-5659-4d72-98f3-d9f954c79488","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:42.645Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"834bb83e-e106-4f79-a080-668ec0af7d87","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:42.645Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"834bb83e-e106-4f79-a080-668ec0af7d87","pod":"fail-88qlb-head-k2rbf","ip":"10.133.0.26"}
{"level":"info","ts":"2026-06-11T10:20:42.645Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"834bb83e-e106-4f79-a080-668ec0af7d87","pod":"fail-88qlb-small-group-worker-p7c6v","ip":"10.134.0.35"}
{"level":"info","ts":"2026-06-11T10:20:42.645Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"834bb83e-e106-4f79-a080-668ec0af7d87","rayCluster":"fail-88qlb","podIPs":["10.133.0.26","10.134.0.35"]}
{"level":"info","ts":"2026-06-11T10:20:45.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"0f96ef40-4f03-4cde-b2d1-6f9d13660ddf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:45.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"0f96ef40-4f03-4cde-b2d1-6f9d13660ddf","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:45.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"0f96ef40-4f03-4cde-b2d1-6f9d13660ddf","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:45.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"0f96ef40-4f03-4cde-b2d1-6f9d13660ddf"}
{"level":"info","ts":"2026-06-11T10:20:45.312Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"0f96ef40-4f03-4cde-b2d1-6f9d13660ddf","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:48.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"62d1a997-41ff-46c6-8385-cfe1a267372f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:48.313Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"62d1a997-41ff-46c6-8385-cfe1a267372f","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:48.313Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"62d1a997-41ff-46c6-8385-cfe1a267372f","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:48.313Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"62d1a997-41ff-46c6-8385-cfe1a267372f"}
{"level":"info","ts":"2026-06-11T10:20:48.313Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"62d1a997-41ff-46c6-8385-cfe1a267372f","RayCluster":"fail-88qlb","State":""}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6"}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6"}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:20:49.922Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"ae85f77a-83cc-48f7-8e7c-bd30ccfd66f6","name":"fail-88qlb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"579d148e-f336-4c48-991e-9af159e54273","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"220fcf56-2a8e-4a1f-b9f4-5a363cc69565","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"579d148e-f336-4c48-991e-9af159e54273","pod":"fail-88qlb-head-k2rbf","ip":"10.133.0.26"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"579d148e-f336-4c48-991e-9af159e54273","pod":"fail-88qlb-small-group-worker-p7c6v","ip":"10.134.0.35"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"579d148e-f336-4c48-991e-9af159e54273","rayCluster":"fail-88qlb","podIPs":["10.133.0.26","10.134.0.35"]}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","head service name":"fail-88qlb-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","head service URL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411"}
{"level":"info","ts":"2026-06-11T10:20:49.938Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","command":["if","!","ray","job","status","--address","http://fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","fail-mjv4l",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","--no-wait","--submission-id","fail-mjv4l","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","--follow","fail-mjv4l"]}
{"level":"info","ts":"2026-06-11T10:20:49.948Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-11T10:20:49.948Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","SubmissionMode":"K8sJobMode","RayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.948Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:20:49.948Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"086670c5-9fcf-4b51-8b6a-48012e27a411","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:20:49.956Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"220fcf56-2a8e-4a1f-b9f4-5a363cc69565","name":"fail-88qlb-head"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","head service name":"fail-88qlb-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","head service URL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","SubmissionMode":"K8sJobMode","RayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:20:49.961Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:20:49.968Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"220fcf56-2a8e-4a1f-b9f4-5a363cc69565","name":"fail-88qlb-workers"}
{"level":"info","ts":"2026-06-11T10:20:49.968Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"220fcf56-2a8e-4a1f-b9f4-5a363cc69565","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.969Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:20:49.969Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0"}
{"level":"error","ts":"2026-06-11T10:20:49.969Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"4ee2f3a2-4b37-4e86-bc5b-c46b2d7d9fa0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:49.969Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"46a3a546-f642-4b61-a32f-dbfc4460a36b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:49.969Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"46a3a546-f642-4b61-a32f-dbfc4460a36b","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.969Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"46a3a546-f642-4b61-a32f-dbfc4460a36b","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:49.969Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"46a3a546-f642-4b61-a32f-dbfc4460a36b"}
{"level":"error","ts":"2026-06-11T10:20:49.978Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"46a3a546-f642-4b61-a32f-dbfc4460a36b","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:49.978Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"46a3a546-f642-4b61-a32f-dbfc4460a36b"}
{"level":"error","ts":"2026-06-11T10:20:49.978Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"46a3a546-f642-4b61-a32f-dbfc4460a36b","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:49.978Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"08cf812f-3456-480f-bc67-e0845eed98f9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:49.978Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"08cf812f-3456-480f-bc67-e0845eed98f9","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:49.978Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"08cf812f-3456-480f-bc67-e0845eed98f9","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:49.978Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"08cf812f-3456-480f-bc67-e0845eed98f9"}
{"level":"error","ts":"2026-06-11T10:20:50.027Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"08cf812f-3456-480f-bc67-e0845eed98f9","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.027Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"08cf812f-3456-480f-bc67-e0845eed98f9"}
{"level":"error","ts":"2026-06-11T10:20:50.027Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"08cf812f-3456-480f-bc67-e0845eed98f9","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.027Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5513bd6b-5d24-4b80-8c91-965920daca3d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:50.027Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5513bd6b-5d24-4b80-8c91-965920daca3d","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:50.027Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5513bd6b-5d24-4b80-8c91-965920daca3d","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:50.028Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5513bd6b-5d24-4b80-8c91-965920daca3d"}
{"level":"error","ts":"2026-06-11T10:20:50.130Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5513bd6b-5d24-4b80-8c91-965920daca3d","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.131Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5513bd6b-5d24-4b80-8c91-965920daca3d"}
{"level":"error","ts":"2026-06-11T10:20:50.131Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5513bd6b-5d24-4b80-8c91-965920daca3d","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"8acb7f74-4f03-4544-94e8-560b074f1c25","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:50.131Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"8acb7f74-4f03-4544-94e8-560b074f1c25","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:50.131Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"8acb7f74-4f03-4544-94e8-560b074f1c25","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:50.131Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"8acb7f74-4f03-4544-94e8-560b074f1c25"}
{"level":"error","ts":"2026-06-11T10:20:50.136Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"8acb7f74-4f03-4544-94e8-560b074f1c25","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.136Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"8acb7f74-4f03-4544-94e8-560b074f1c25"}
{"level":"error","ts":"2026-06-11T10:20:50.136Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"8acb7f74-4f03-4544-94e8-560b074f1c25","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.171Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a8a5777a-730c-4eb9-bda9-64317daf7d94","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:50.171Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a8a5777a-730c-4eb9-bda9-64317daf7d94","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:50.171Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a8a5777a-730c-4eb9-bda9-64317daf7d94","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:50.171Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a8a5777a-730c-4eb9-bda9-64317daf7d94"}
{"level":"error","ts":"2026-06-11T10:20:50.177Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a8a5777a-730c-4eb9-bda9-64317daf7d94","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.177Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a8a5777a-730c-4eb9-bda9-64317daf7d94"}
{"level":"error","ts":"2026-06-11T10:20:50.177Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a8a5777a-730c-4eb9-bda9-64317daf7d94","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"edc95d11-7421-45ff-8086-b257a628237b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:50.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"edc95d11-7421-45ff-8086-b257a628237b","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:50.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"edc95d11-7421-45ff-8086-b257a628237b","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:50.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"edc95d11-7421-45ff-8086-b257a628237b"}
{"level":"error","ts":"2026-06-11T10:20:50.343Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"edc95d11-7421-45ff-8086-b257a628237b","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.344Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"edc95d11-7421-45ff-8086-b257a628237b"}
{"level":"error","ts":"2026-06-11T10:20:50.344Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"edc95d11-7421-45ff-8086-b257a628237b","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.664Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7c133352-0ae3-407f-8b2f-c053d66de52b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:50.664Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7c133352-0ae3-407f-8b2f-c053d66de52b","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:50.664Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7c133352-0ae3-407f-8b2f-c053d66de52b","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:50.664Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7c133352-0ae3-407f-8b2f-c053d66de52b"}
{"level":"error","ts":"2026-06-11T10:20:50.670Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7c133352-0ae3-407f-8b2f-c053d66de52b","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.671Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7c133352-0ae3-407f-8b2f-c053d66de52b"}
{"level":"error","ts":"2026-06-11T10:20:50.671Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7c133352-0ae3-407f-8b2f-c053d66de52b","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.976Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"510814d5-a011-4813-b661-ec634f7c9ae3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:50.976Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"510814d5-a011-4813-b661-ec634f7c9ae3","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:50.976Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"510814d5-a011-4813-b661-ec634f7c9ae3","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:50.977Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"510814d5-a011-4813-b661-ec634f7c9ae3"}
{"level":"error","ts":"2026-06-11T10:20:50.982Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"510814d5-a011-4813-b661-ec634f7c9ae3","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:50.982Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"510814d5-a011-4813-b661-ec634f7c9ae3"}
{"level":"error","ts":"2026-06-11T10:20:50.982Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"510814d5-a011-4813-b661-ec634f7c9ae3","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:51.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f0df1f98-b97b-4a3d-81b5-5bbb2640fc0a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:51.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f0df1f98-b97b-4a3d-81b5-5bbb2640fc0a","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:51.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f0df1f98-b97b-4a3d-81b5-5bbb2640fc0a","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:51.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f0df1f98-b97b-4a3d-81b5-5bbb2640fc0a"}
{"level":"error","ts":"2026-06-11T10:20:51.317Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f0df1f98-b97b-4a3d-81b5-5bbb2640fc0a","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:51.317Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f0df1f98-b97b-4a3d-81b5-5bbb2640fc0a"}
{"level":"error","ts":"2026-06-11T10:20:51.317Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"f0df1f98-b97b-4a3d-81b5-5bbb2640fc0a","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:51.938Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47"}
{"level":"info","ts":"2026-06-11T10:20:51.938Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47"}
{"level":"info","ts":"2026-06-11T10:20:51.938Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","1 head service found":"fail-88qlb-head-svc"}
{"level":"info","ts":"2026-06-11T10:20:51.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","Found 1 head Pod":"fail-88qlb-head-k2rbf","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:20:51.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","head Pod":"fail-88qlb-head-k2rbf","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-88qlb-head-k2rbf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:51.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:20:51.939Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","worker Pod":"fail-88qlb-small-group-worker-p7c6v","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-88qlb-small-group-worker-p7c6v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:20:51.939Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:51.939Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:20:51.939Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:20:51.939Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:20:51.939Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"7e08094f-4dda-4d07-a2f1-5bdc9ae2ad47","seconds":300}
{"level":"info","ts":"2026-06-11T10:20:53.877Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"9ee4cac1-8bf7-4271-80ec-68a8d5673cb2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:53.877Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"9ee4cac1-8bf7-4271-80ec-68a8d5673cb2","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:53.877Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"9ee4cac1-8bf7-4271-80ec-68a8d5673cb2","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:53.877Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"9ee4cac1-8bf7-4271-80ec-68a8d5673cb2"}
{"level":"error","ts":"2026-06-11T10:20:53.883Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"9ee4cac1-8bf7-4271-80ec-68a8d5673cb2","JobId":"fail-mjv4l","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:53.883Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"9ee4cac1-8bf7-4271-80ec-68a8d5673cb2"}
{"level":"error","ts":"2026-06-11T10:20:53.883Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"9ee4cac1-8bf7-4271-80ec-68a8d5673cb2","error":"Job fail-mjv4l does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:20:59.003Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e17fcc02-b99f-4b1f-93b9-eab41a0c32f9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:59.003Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e17fcc02-b99f-4b1f-93b9-eab41a0c32f9","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:59.004Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e17fcc02-b99f-4b1f-93b9-eab41a0c32f9","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:59.004Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e17fcc02-b99f-4b1f-93b9-eab41a0c32f9"}
{"level":"info","ts":"2026-06-11T10:20:59.009Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e17fcc02-b99f-4b1f-93b9-eab41a0c32f9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:20:59.010Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e17fcc02-b99f-4b1f-93b9-eab41a0c32f9","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:20:59.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e658be7a-eb14-49ff-a804-2adc96840edf","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:20:59.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e658be7a-eb14-49ff-a804-2adc96840edf","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:20:59.026Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e658be7a-eb14-49ff-a804-2adc96840edf","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:20:59.026Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e658be7a-eb14-49ff-a804-2adc96840edf"}
{"level":"info","ts":"2026-06-11T10:20:59.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e658be7a-eb14-49ff-a804-2adc96840edf","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:02.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6c00ea63-262a-4d4a-b130-488bc3b88596","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:02.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6c00ea63-262a-4d4a-b130-488bc3b88596","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:02.026Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6c00ea63-262a-4d4a-b130-488bc3b88596","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:02.027Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6c00ea63-262a-4d4a-b130-488bc3b88596"}
{"level":"info","ts":"2026-06-11T10:21:02.030Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6c00ea63-262a-4d4a-b130-488bc3b88596","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:02.030Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"6c00ea63-262a-4d4a-b130-488bc3b88596","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:21:02.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1bf63689-3f03-4338-ac52-f2aae38c4e3d","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:02.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1bf63689-3f03-4338-ac52-f2aae38c4e3d","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:02.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1bf63689-3f03-4338-ac52-f2aae38c4e3d","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:02.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1bf63689-3f03-4338-ac52-f2aae38c4e3d"}
{"level":"info","ts":"2026-06-11T10:21:02.049Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"1bf63689-3f03-4338-ac52-f2aae38c4e3d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:05.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"850c5682-26db-4e35-9829-ccd1bb417187","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:05.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"850c5682-26db-4e35-9829-ccd1bb417187","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:05.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"850c5682-26db-4e35-9829-ccd1bb417187","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:05.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"850c5682-26db-4e35-9829-ccd1bb417187"}
{"level":"info","ts":"2026-06-11T10:21:05.049Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"850c5682-26db-4e35-9829-ccd1bb417187","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:08.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d276cbc2-345c-4bba-ba4b-78e41b91b15c","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:08.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d276cbc2-345c-4bba-ba4b-78e41b91b15c","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:08.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d276cbc2-345c-4bba-ba4b-78e41b91b15c","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:08.022Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d276cbc2-345c-4bba-ba4b-78e41b91b15c"}
{"level":"info","ts":"2026-06-11T10:21:08.025Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d276cbc2-345c-4bba-ba4b-78e41b91b15c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:08.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d6f11446-22c4-4e74-bb68-a97fa1cea3f1","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:08.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d6f11446-22c4-4e74-bb68-a97fa1cea3f1","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:08.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d6f11446-22c4-4e74-bb68-a97fa1cea3f1","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:08.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d6f11446-22c4-4e74-bb68-a97fa1cea3f1"}
{"level":"info","ts":"2026-06-11T10:21:08.054Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d6f11446-22c4-4e74-bb68-a97fa1cea3f1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:09.031Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fb27cdfa-e2c6-4a3f-a080-9b5a60c4570d","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:09.031Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fb27cdfa-e2c6-4a3f-a080-9b5a60c4570d","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.031Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fb27cdfa-e2c6-4a3f-a080-9b5a60c4570d","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:09.031Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fb27cdfa-e2c6-4a3f-a080-9b5a60c4570d"}
{"level":"info","ts":"2026-06-11T10:21:09.034Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"fb27cdfa-e2c6-4a3f-a080-9b5a60c4570d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:09.053Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"98231617-bafb-4944-8b54-912f2414d9f4","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:09.053Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"98231617-bafb-4944-8b54-912f2414d9f4","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.053Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"98231617-bafb-4944-8b54-912f2414d9f4","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:09.053Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"98231617-bafb-4944-8b54-912f2414d9f4"}
{"level":"info","ts":"2026-06-11T10:21:09.056Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"98231617-bafb-4944-8b54-912f2414d9f4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:20:57Z","endTime":"2026-06-11T10:20:59Z"},"jobId":"fail-mjv4l","rayClusterName":"fail-88qlb","dashboardURL":"fail-88qlb-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:20:57,469\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-mjv4l: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:20:12Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:20:49Z","stateTransitionTimes":{"ready":"2026-06-11T10:20:49Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.26","serviceIP":"10.133.0.26","podName":"fail-88qlb-head-k2rbf","serviceName":"fail-88qlb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:20:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:20:49Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:20:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:20:12Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:21:09.056Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"98231617-bafb-4944-8b54-912f2414d9f4","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-11T10:21:09.073Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5a5b964f-f1b8-41c8-a457-67b8bd0553d5","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:09.073Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5a5b964f-f1b8-41c8-a457-67b8bd0553d5","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:21:09 +0000 UTC","Now":"2026-06-11T10:21:09.073Z","ShutdownTime":"2026-06-11T10:21:09.000Z"}
{"level":"info","ts":"2026-06-11T10:21:09.670Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a990470a-1147-4b76-8f31-650001667b5c","DeletionTimestamp":"2026-06-11 10:21:09 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:21:09.670Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"a990470a-1147-4b76-8f31-650001667b5c","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:21:09.688Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"67349eb2-52c0-4db1-b42d-9a5f1f216ef5"}
{"level":"info","ts":"2026-06-11T10:21:09.705Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"5a45ebb9-f923-4eb5-88d5-9cf1e027011b"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a6ebe98b-428f-48ab-bfa8-4ed9ce46b74a","rayCluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"b31e64fa-065b-4b46-a34c-1d834d93a017"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"reconcileID":"e9890196-b199-4861-bf31-2b85ed085114"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"d4992dd0-182e-4596-816b-6aef905443fb"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a6ebe98b-428f-48ab-bfa8-4ed9ce46b74a","pod":"fail-88qlb-head-k2rbf","ip":"10.133.0.26"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a6ebe98b-428f-48ab-bfa8-4ed9ce46b74a","pod":"fail-88qlb-small-group-worker-p7c6v","ip":"10.134.0.35"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:21:09.711Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a6ebe98b-428f-48ab-bfa8-4ed9ce46b74a","rayCluster":"fail-88qlb","podIPs":["10.133.0.26","10.134.0.35"]}
{"level":"info","ts":"2026-06-11T10:21:09.721Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","name":"kuberay-gateway-access","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:09.727Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","name":"test-ns-c6srn-fail-88qlb","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:21:09.733Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","configMap":"kube-rbac-proxy-config-fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.738Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","serviceAccount":"fail-88qlb-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:21:09.746Z","logger":"raycluster-resource","msg":"default","name":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.747Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-88qlb","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:09.756Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"1b190fe5-fc62-40d3-825d-32114b250e10","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.756Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:09.757Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.757Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f","cluster":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.757Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:21:09.757Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:21:09.757Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"f0db003a-f828-4be3-90ea-516f238ec65a","namespace":"test-ns-c6srn","clusterName":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:09.757Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"81a02540-88b2-405c-87c8-9ad1e2d44a66"}
{"level":"info","ts":"2026-06-11T10:21:09.757Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"e9bc368b-60da-4381-b923-28e69967c5c4"}
{"level":"error","ts":"2026-06-11T10:21:09.762Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f","error":"rayclusters.ray.io \"fail-88qlb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:09.762Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f"}
{"level":"error","ts":"2026-06-11T10:21:09.762Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a5ac51ac-cb05-4f43-945f-6ac302a41e7f","error":"rayclusters.ray.io \"fail-88qlb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:09.762Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"64ce697a-da8d-4b52-9947-74d021c886c0","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:09.762Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"64ce697a-da8d-4b52-9947-74d021c886c0"}
{"level":"info","ts":"2026-06-11T10:21:09.762Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"64ce697a-da8d-4b52-9947-74d021c886c0","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:21:09.762Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"64ce697a-da8d-4b52-9947-74d021c886c0","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:09.766Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"aca09205-4159-47d4-b704-580bc24951e8"}
{"level":"info","ts":"2026-06-11T10:21:09.768Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"85cbbb3c-2968-4d66-9539-0ef868333fcc","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:09.768Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"85cbbb3c-2968-4d66-9539-0ef868333fcc"}
{"level":"info","ts":"2026-06-11T10:21:09.768Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"85cbbb3c-2968-4d66-9539-0ef868333fcc","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:21:09.768Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"85cbbb3c-2968-4d66-9539-0ef868333fcc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:09.786Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"fdd78fff-7d65-4a8c-ab9f-acdfd2eca434"}
{"level":"info","ts":"2026-06-11T10:21:09.791Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"6762917e-bfd5-4d0f-bc43-380d2f071121","namespacedName":{"name":"fail-88qlb","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:09.791Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"6762917e-bfd5-4d0f-bc43-380d2f071121"}
{"level":"info","ts":"2026-06-11T10:21:09.791Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"6762917e-bfd5-4d0f-bc43-380d2f071121","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:21:09.791Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"6762917e-bfd5-4d0f-bc43-380d2f071121","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:09.847Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"f0db003a-f828-4be3-90ea-516f238ec65a","namespace":"test-ns-c6srn","clusterName":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:10.702Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9765a371-625d-4ef9-8783-02f83d129d3e","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:10.703Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9765a371-625d-4ef9-8783-02f83d129d3e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:21:10.717Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9765a371-625d-4ef9-8783-02f83d129d3e"}
{"level":"info","ts":"2026-06-11T10:21:10.717Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9765a371-625d-4ef9-8783-02f83d129d3e","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:21:10.717Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9765a371-625d-4ef9-8783-02f83d129d3e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-8gn4s","rayClusterName":"fail-k8s-job-b6vx5","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:21:10.717Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9765a371-625d-4ef9-8783-02f83d129d3e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:21:10.736Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:10.736Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf"}
{"level":"info","ts":"2026-06-11T10:21:10.736Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:21:10.736Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-z7k2w","rayClusterName":"fail-k8s-job-5x4qs","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:21:10.736Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:21:10.749Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:21:10.749Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf"}
{"level":"error","ts":"2026-06-11T10:21:10.749Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c461af1e-3544-4333-8497-57bc2316d5bf","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:10.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f65b9abf-88cc-473e-a0ac-2aaeba0f9bec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:10.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f65b9abf-88cc-473e-a0ac-2aaeba0f9bec","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.749Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f65b9abf-88cc-473e-a0ac-2aaeba0f9bec","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:10.749Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f65b9abf-88cc-473e-a0ac-2aaeba0f9bec","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:10.758Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.758Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20b28dbc-1be4-4a89-9d67-e5f7026483cc","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0"}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0"}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","mode":"IntegratedOAuth","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.782Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.781Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.782Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f65b9abf-88cc-473e-a0ac-2aaeba0f9bec","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:10.782Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f65b9abf-88cc-473e-a0ac-2aaeba0f9bec"}
{"level":"info","ts":"2026-06-11T10:21:10.782Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f65b9abf-88cc-473e-a0ac-2aaeba0f9bec","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:10.783Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"77458220-29b1-47d4-bc96-932ea4bda9ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:10.783Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"77458220-29b1-47d4-bc96-932ea4bda9ee","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.783Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"77458220-29b1-47d4-bc96-932ea4bda9ee","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:10.783Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"77458220-29b1-47d4-bc96-932ea4bda9ee"}
{"level":"info","ts":"2026-06-11T10:21:10.783Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"77458220-29b1-47d4-bc96-932ea4bda9ee","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:10.788Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.788Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","name":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:21:10.792Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:10.795Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.795Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.798Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e0e4c8fe-d348-485e-b710-643a411674e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:10.798Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e0e4c8fe-d348-485e-b710-643a411674e2","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.798Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e0e4c8fe-d348-485e-b710-643a411674e2","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:10.798Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e0e4c8fe-d348-485e-b710-643a411674e2"}
{"level":"info","ts":"2026-06-11T10:21:10.798Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e0e4c8fe-d348-485e-b710-643a411674e2","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:10.801Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20b28dbc-1be4-4a89-9d67-e5f7026483cc","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.801Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"9dad8f62-aec1-435a-857d-e962fb32ceba","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","name":"fail-k8s-job-b6vx5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-b6vx5\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5"}}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0"}
{"level":"error","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e83bd2c9-6043-4e84-b88d-839a7eb88fa0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105"}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105"}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:21:10.806Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:10.827Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c"}
{"level":"info","ts":"2026-06-11T10:21:10.827Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:21:10.827Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.827Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","name":"fail-k8s-job-b6vx5-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105"}
{"level":"error","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4b0b0564-96da-4912-83a0-93858948c105","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ec594bc2-777d-4089-8c7f-a6d23cdd6efa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ec594bc2-777d-4089-8c7f-a6d23cdd6efa","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ec594bc2-777d-4089-8c7f-a6d23cdd6efa","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29"}
{"level":"info","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29"}
{"level":"info","ts":"2026-06-11T10:21:10.828Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29"}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","name":"kuberay-gateway-access","namespace":"test-ns-c6srn","operation":"created"}
{"level":"info","ts":"2026-06-11T10:21:10.829Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ec594bc2-777d-4089-8c7f-a6d23cdd6efa"}
{"level":"info","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ec594bc2-777d-4089-8c7f-a6d23cdd6efa","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:10.928Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"9dad8f62-aec1-435a-857d-e962fb32ceba","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:10.928Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"error","ts":"2026-06-11T10:21:10.927Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b0cc7ea7-6434-47c5-a433-6d0f2a8b5c29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","name":"test-ns-c6srn-fail-k8s-job-b6vx5","operation":"created"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c"}
{"level":"info","ts":"2026-06-11T10:21:11.027Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858"}
{"level":"error","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"d2c13169-0710-4bcc-a6d0-7fa03256d858","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa"}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa"}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa"}
{"level":"error","ts":"2026-06-11T10:21:11.028Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"207088eb-358b-44f7-8129-5e5afc9a21fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-c6srn"},"reconcileID":"7f5a1fd3-563c-49aa-9d6b-1e64798ed937"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","name":"kube-rbac-proxy-config-fail-k8s-job-b6vx5","operation":"created"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:11.127Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","rayCluster":"fail-k8s-job-b6vx5","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:21:11.128Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-k8s-job-b6vx5\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c"}
{"level":"error","ts":"2026-06-11T10:21:11.128Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffc04bed-3304-4002-b184-18a8ce0c4d2c","error":"Certificate.cert-manager.io \"ray-worker-cert-fail-k8s-job-b6vx5\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"37db3a6b-3ba3-4a6a-8a6f-9310f40daa5b","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542"}
{"level":"error","ts":"2026-06-11T10:21:11.128Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"e700510f-01a8-4b30-adbe-3cec84fdc542","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.127Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"9dad8f62-aec1-435a-857d-e962fb32ceba","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"9dad8f62-aec1-435a-857d-e962fb32ceba","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"37db3a6b-3ba3-4a6a-8a6f-9310f40daa5b"}
{"level":"info","ts":"2026-06-11T10:21:11.128Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"55f6993f-208c-4da4-9cc5-348c9aad758b","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.133Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"533a04a2-ae13-44d5-b535-1e232fccd70a","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.133Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"533a04a2-ae13-44d5-b535-1e232fccd70a"}
{"level":"error","ts":"2026-06-11T10:21:11.136Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-b6vx5\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.136Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc"}
{"level":"error","ts":"2026-06-11T10:21:11.136Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"20d7da78-5b5a-49c4-b521-b46377259adc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-b6vx5\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.136Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"96a82a15-d5e0-45e0-ace3-7898779f5fa5","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:11.136Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"96a82a15-d5e0-45e0-ace3-7898779f5fa5","mode":"IntegratedOAuth","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.136Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"96a82a15-d5e0-45e0-ace3-7898779f5fa5","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"96a82a15-d5e0-45e0-ace3-7898779f5fa5","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"7eb30c36-2fd8-42a6-8d52-fa6029f6df89","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"68fa9e79-b380-4791-9edc-63a92c1caacd","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"68fa9e79-b380-4791-9edc-63a92c1caacd","mode":"IntegratedOAuth","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"68fa9e79-b380-4791-9edc-63a92c1caacd","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"7eb30c36-2fd8-42a6-8d52-fa6029f6df89"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"616ae28c-5201-4ca6-9330-1e5051725a18","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"616ae28c-5201-4ca6-9330-1e5051725a18","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"616ae28c-5201-4ca6-9330-1e5051725a18","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"616ae28c-5201-4ca6-9330-1e5051725a18"}
{"level":"info","ts":"2026-06-11T10:21:11.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"616ae28c-5201-4ca6-9330-1e5051725a18","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:11.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef"}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef"}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef"}
{"level":"error","ts":"2026-06-11T10:21:11.289Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"db45f052-f9be-42bb-96ce-4893bc5658ef","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.327Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"55f6993f-208c-4da4-9cc5-348c9aad758b","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:11.337Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"68fa9e79-b380-4791-9edc-63a92c1caacd","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"55f6993f-208c-4da4-9cc5-348c9aad758b","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:11.341Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"55f6993f-208c-4da4-9cc5-348c9aad758b","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.341Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"da0103a7-c7f5-4edc-8112-7cd862f78f1e","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.356Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"da0103a7-c7f5-4edc-8112-7cd862f78f1e","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:11.368Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"da0103a7-c7f5-4edc-8112-7cd862f78f1e","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:11.369Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"da0103a7-c7f5-4edc-8112-7cd862f78f1e","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:11.609Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d"}
{"level":"info","ts":"2026-06-11T10:21:11.609Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d"}
{"level":"info","ts":"2026-06-11T10:21:11.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:11.609Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:11.609Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:11.609Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:11.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:11.610Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d"}
{"level":"error","ts":"2026-06-11T10:21:11.610Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5ed3daff-a2e3-4bc6-bc91-190c2442693d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:12.250Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2"}
{"level":"info","ts":"2026-06-11T10:21:12.250Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2"}
{"level":"info","ts":"2026-06-11T10:21:12.250Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:12.250Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","error":"head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"error","ts":"2026-06-11T10:21:12.250Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:12.250Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found"}
{"level":"info","ts":"2026-06-11T10:21:12.251Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:12.251Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:12.251Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","filter labels":["test-ns-c6srn",{"ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:21:12.251Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2"}
{"level":"error","ts":"2026-06-11T10:21:12.251Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4679f6-a082-4503-b7cd-28dc8ddf95d2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-b6vx5 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","mode":"IntegratedOAuth","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","cluster":"fail-k8s-job-b6vx5","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","cluster":"fail-k8s-job-b6vx5","authType":"IntegratedOAuth","serviceAccount":"fail-k8s-job-b6vx5-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-k8s-job-b6vx5","ray.io/group":"headgroup","ray.io/identifier":"fail-k8s-job-b6vx5-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:21:13.531Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:21:13.532Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:21:13.532Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:21:13.532Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","enabled":true}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","name":"fail-k8s-job-b6vx5-head-zn8b8"}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc"}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","nodeType":"worker","rayStartParams":{"address":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","rayStartCmd":"ray start  --address=fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:21:13.550Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","enabled":true}
{"level":"info","ts":"2026-06-11T10:21:13.568Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","name":"fail-k8s-job-b6vx5-small-group-worker-hhh7t"}
{"level":"info","ts":"2026-06-11T10:21:13.568Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.569Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.569Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-b6vx5-head-svc"},"newHead":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"}}
{"level":"info","ts":"2026-06-11T10:21:13.569Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8010e014-a336-4d27-b5fa-fbdc5a9b8ecc","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59"}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59"}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:13.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"4c885c36-dbbc-4682-b6eb-91f9cc5b83f9","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"6d4f5310-f910-428c-a497-6ff5600c87c2","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-b6vx5-head-svc"},"newHead":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"}}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f8037591-210b-4ad4-96e2-a794b6c0bf34","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f8037591-210b-4ad4-96e2-a794b6c0bf34","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f8037591-210b-4ad4-96e2-a794b6c0bf34","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:13.582Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"6d4f5310-f910-428c-a497-6ff5600c87c2","rayCluster":"fail-k8s-job-b6vx5","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f8037591-210b-4ad4-96e2-a794b6c0bf34"}
{"level":"info","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"f8037591-210b-4ad4-96e2-a794b6c0bf34","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","name":"fail-k8s-job-b6vx5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-b6vx5\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5"}}
{"level":"info","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59"}
{"level":"error","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"36d25ec4-819d-46f6-ae4f-142d0aba3f59","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-b6vx5\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c"}
{"level":"info","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c"}
{"level":"info","ts":"2026-06-11T10:21:13.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:21:13.628Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"19946780-04af-4513-90e0-42aa9553dd8c","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:13.633Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"4c885c36-dbbc-4682-b6eb-91f9cc5b83f9","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:21:13.641Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:13.642Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0d5d7d35-8a1f-463a-b4a3-2afdfd5e56af","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"1dac5107-89dd-46ea-8aff-f6e9095b997c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:13.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"1dac5107-89dd-46ea-8aff-f6e9095b997c","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"1dac5107-89dd-46ea-8aff-f6e9095b997c","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:13.642Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0d5d7d35-8a1f-463a-b4a3-2afdfd5e56af","rayCluster":"fail-k8s-job-b6vx5","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:21:13.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"1dac5107-89dd-46ea-8aff-f6e9095b997c"}
{"level":"info","ts":"2026-06-11T10:21:13.642Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"1dac5107-89dd-46ea-8aff-f6e9095b997c","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:13.645Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"4c885c36-dbbc-4682-b6eb-91f9cc5b83f9","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:13.645Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"4c885c36-dbbc-4682-b6eb-91f9cc5b83f9","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.645Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"aa1cfe0c-42dc-4f60-a93b-f61a0fbba608","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.649Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","name":"fail-k8s-job-b6vx5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-b6vx5\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5"}}
{"level":"info","ts":"2026-06-11T10:21:13.649Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b"}
{"level":"error","ts":"2026-06-11T10:21:13.649Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"12e1da26-23f6-4e49-9ef1-b0084d9b8c7b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-b6vx5\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.654Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:13.655Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:13.655Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"3116fd0c-68bf-4214-b348-b92b6ef8f556","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:13.656Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"aa1cfe0c-42dc-4f60-a93b-f61a0fbba608","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:13.668Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"aa1cfe0c-42dc-4f60-a93b-f61a0fbba608","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:13.668Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"aa1cfe0c-42dc-4f60-a93b-f61a0fbba608","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.783Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7d72e99b-f6c0-4a17-be07-d7d1e3b0ed28","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:13.783Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7d72e99b-f6c0-4a17-be07-d7d1e3b0ed28","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:13.783Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7d72e99b-f6c0-4a17-be07-d7d1e3b0ed28","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:13.784Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7d72e99b-f6c0-4a17-be07-d7d1e3b0ed28"}
{"level":"info","ts":"2026-06-11T10:21:13.784Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7d72e99b-f6c0-4a17-be07-d7d1e3b0ed28","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:14.018Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:14.019Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"c3a218a0-1010-4333-9f32-f10abe1d54a1","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:14.053Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518"}
{"level":"info","ts":"2026-06-11T10:21:14.053Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518"}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:14.054Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"8bcf7c11-6c1a-4ee4-8f04-f5092d597518","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"newHead":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"}}
{"level":"info","ts":"2026-06-11T10:21:14.623Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"05cb3e34-d227-405b-b508-5df919a6455d","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"46829bde-1ce4-4d3a-aac9-74600e3a3414","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3132911e-2aec-4723-a8ea-1ba9fcaa9038","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8d39621-aff2-435a-b350-1cb4dd7c602f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8d39621-aff2-435a-b350-1cb4dd7c602f","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8d39621-aff2-435a-b350-1cb4dd7c602f","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"46829bde-1ce4-4d3a-aac9-74600e3a3414","pod":"fail-k8s-job-b6vx5-head-zn8b8","ip":"10.133.0.27"}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8d39621-aff2-435a-b350-1cb4dd7c602f"}
{"level":"info","ts":"2026-06-11T10:21:14.649Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8d39621-aff2-435a-b350-1cb4dd7c602f","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:14.667Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"46829bde-1ce4-4d3a-aac9-74600e3a3414","certificate":"ray-head-cert-fail-k8s-job-b6vx5","dnsNames":["fail-k8s-job-b6vx5-head-svc","localhost","fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc","fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.27","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:21:14.677Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3132911e-2aec-4723-a8ea-1ba9fcaa9038","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:14.693Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"46829bde-1ce4-4d3a-aac9-74600e3a3414","certificate":"ray-worker-cert-fail-k8s-job-b6vx5","dnsNames":["fail-k8s-job-b6vx5-worker-svc","localhost","fail-k8s-job-b6vx5-small-group","fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc","fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc.cluster.local","*.fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc","*.fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.27","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:21:14.693Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"46829bde-1ce4-4d3a-aac9-74600e3a3414","rayCluster":"fail-k8s-job-b6vx5","podIPs":["10.133.0.27"]}
{"level":"info","ts":"2026-06-11T10:21:14.701Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3132911e-2aec-4723-a8ea-1ba9fcaa9038","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:14.701Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3132911e-2aec-4723-a8ea-1ba9fcaa9038","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:15.050Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c"}
{"level":"info","ts":"2026-06-11T10:21:15.050Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c"}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:15.051Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"4ae2c592-1306-492b-b509-557bf2cfa53c","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:16.586Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43f2fee7-ba4f-4511-8c41-fd34dd8ebd82","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:21:16.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43f2fee7-ba4f-4511-8c41-fd34dd8ebd82","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:21:16.586Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43f2fee7-ba4f-4511-8c41-fd34dd8ebd82","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:21:16.586Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"43f2fee7-ba4f-4511-8c41-fd34dd8ebd82","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:16.648Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a1cd40fa-28ee-4a82-8157-8f71edcff927","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:16.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"85ed187f-7ad9-42f2-91dc-3fc5a08bbf69","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:16.784Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"85ed187f-7ad9-42f2-91dc-3fc5a08bbf69","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:16.785Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"85ed187f-7ad9-42f2-91dc-3fc5a08bbf69","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:16.785Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"85ed187f-7ad9-42f2-91dc-3fc5a08bbf69"}
{"level":"info","ts":"2026-06-11T10:21:16.785Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"85ed187f-7ad9-42f2-91dc-3fc5a08bbf69","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:19.629Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"72b456b2-5f89-4513-8d44-96f44ef41bb8","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:19.785Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4afa3605-0c7b-44b3-ac5f-fc380cd557e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:19.785Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4afa3605-0c7b-44b3-ac5f-fc380cd557e7","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:19.785Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4afa3605-0c7b-44b3-ac5f-fc380cd557e7","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:19.785Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4afa3605-0c7b-44b3-ac5f-fc380cd557e7"}
{"level":"info","ts":"2026-06-11T10:21:19.785Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4afa3605-0c7b-44b3-ac5f-fc380cd557e7","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:20.637Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae"}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae"}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:21:20.638Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"97da3409-053d-471e-b12f-9e058882f5ae","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0e227e23-255b-47cb-a62e-f8502c10d50e","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"2dd14fdd-d3a3-46b3-a362-67a60f2796e6","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0e227e23-255b-47cb-a62e-f8502c10d50e","pod":"fail-k8s-job-b6vx5-head-zn8b8","ip":"10.133.0.27"}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0e227e23-255b-47cb-a62e-f8502c10d50e","pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","ip":"10.134.0.37"}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b8b05aa9-0dcd-43f5-8533-5db7c6eb9ee0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b8b05aa9-0dcd-43f5-8533-5db7c6eb9ee0","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b8b05aa9-0dcd-43f5-8533-5db7c6eb9ee0","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b8b05aa9-0dcd-43f5-8533-5db7c6eb9ee0"}
{"level":"info","ts":"2026-06-11T10:21:20.652Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b8b05aa9-0dcd-43f5-8533-5db7c6eb9ee0","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:20.665Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0e227e23-255b-47cb-a62e-f8502c10d50e","certificate":"ray-head-cert-fail-k8s-job-b6vx5","dnsNames":["fail-k8s-job-b6vx5-head-svc","localhost","fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc","fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.27","10.134.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:21:20.666Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"2dd14fdd-d3a3-46b3-a362-67a60f2796e6","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:20.679Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0e227e23-255b-47cb-a62e-f8502c10d50e","certificate":"ray-worker-cert-fail-k8s-job-b6vx5","dnsNames":["fail-k8s-job-b6vx5-worker-svc","localhost","fail-k8s-job-b6vx5-small-group","fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc","fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc.cluster.local","*.fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc","*.fail-k8s-job-b6vx5-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.27","10.134.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:21:20.679Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"0e227e23-255b-47cb-a62e-f8502c10d50e","rayCluster":"fail-k8s-job-b6vx5","podIPs":["10.133.0.27","10.134.0.37"]}
{"level":"info","ts":"2026-06-11T10:21:20.680Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"2dd14fdd-d3a3-46b3-a362-67a60f2796e6","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:20.681Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"2dd14fdd-d3a3-46b3-a362-67a60f2796e6","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:22.652Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"30412235-f596-4336-be8d-84d989a73ddf","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:22.786Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"52b6aecd-0651-4fce-92d7-35f3aaeaaeea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:22.786Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"52b6aecd-0651-4fce-92d7-35f3aaeaaeea","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:22.786Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"52b6aecd-0651-4fce-92d7-35f3aaeaaeea","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:22.786Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"52b6aecd-0651-4fce-92d7-35f3aaeaaeea"}
{"level":"info","ts":"2026-06-11T10:21:22.786Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"52b6aecd-0651-4fce-92d7-35f3aaeaaeea","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:24.091Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:24.092Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"b748e173-63f9-4d48-90c0-10f1a68b3747","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:25.102Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6"}
{"level":"info","ts":"2026-06-11T10:21:25.102Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6"}
{"level":"info","ts":"2026-06-11T10:21:25.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","podName":"fail-k8s-job-b6vx5-head-zn8b8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:21:25.103Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"a174d18e-ea12-4423-a2a8-e233ff0d16a6","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d95c4652-aba9-44aa-a212-90defdef029d","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"03f411ea-0669-40fc-a498-ae9e78bbb23e","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d95c4652-aba9-44aa-a212-90defdef029d","pod":"fail-k8s-job-b6vx5-head-zn8b8","ip":"10.133.0.27"}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d95c4652-aba9-44aa-a212-90defdef029d","pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","ip":"10.134.0.37"}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d95c4652-aba9-44aa-a212-90defdef029d","rayCluster":"fail-k8s-job-b6vx5","podIPs":["10.133.0.27","10.134.0.37"]}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"aca87c66-7a53-4267-9913-9473413618a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"aca87c66-7a53-4267-9913-9473413618a8","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"aca87c66-7a53-4267-9913-9473413618a8","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"aca87c66-7a53-4267-9913-9473413618a8"}
{"level":"info","ts":"2026-06-11T10:21:25.126Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"aca87c66-7a53-4267-9913-9473413618a8","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:25.141Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"03f411ea-0669-40fc-a498-ae9e78bbb23e","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:25.158Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"03f411ea-0669-40fc-a498-ae9e78bbb23e","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:25.158Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"03f411ea-0669-40fc-a498-ae9e78bbb23e","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:25.787Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"3948de90-fd81-45e9-9228-93157b1e7261","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:25.787Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"3948de90-fd81-45e9-9228-93157b1e7261","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:25.787Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"3948de90-fd81-45e9-9228-93157b1e7261","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:25.787Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"3948de90-fd81-45e9-9228-93157b1e7261"}
{"level":"info","ts":"2026-06-11T10:21:25.787Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"3948de90-fd81-45e9-9228-93157b1e7261","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6"}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6"}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:27.126Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"60dad3b7-4937-41c2-971f-030b38684eb6","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:28.788Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"215af9ce-8665-4673-818a-4d06721e8eca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:28.788Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"215af9ce-8665-4673-818a-4d06721e8eca","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:28.788Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"215af9ce-8665-4673-818a-4d06721e8eca","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:28.788Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"215af9ce-8665-4673-818a-4d06721e8eca"}
{"level":"info","ts":"2026-06-11T10:21:28.788Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"215af9ce-8665-4673-818a-4d06721e8eca","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240"}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240"}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:31.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:31.670Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:21:31.670Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:21:31.670Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:21:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:21:31.670Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"ac396151-1c03-4f42-a949-91043ac7b240","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:21:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d7d76a09-9d0e-4f2f-b188-25de9330a975","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"a2fbfcab-726e-40fe-bb97-b7bbeb4e7dbf","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"a2fbfcab-726e-40fe-bb97-b7bbeb4e7dbf","pod":"fail-k8s-job-b6vx5-head-zn8b8","ip":"10.133.0.27"}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"a2fbfcab-726e-40fe-bb97-b7bbeb4e7dbf","pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","ip":"10.134.0.37"}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9929935e-f69b-40ba-80ec-347dcee30796","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9929935e-f69b-40ba-80ec-347dcee30796","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9929935e-f69b-40ba-80ec-347dcee30796","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"a2fbfcab-726e-40fe-bb97-b7bbeb4e7dbf","rayCluster":"fail-k8s-job-b6vx5","podIPs":["10.133.0.27","10.134.0.37"]}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9929935e-f69b-40ba-80ec-347dcee30796"}
{"level":"info","ts":"2026-06-11T10:21:31.686Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9929935e-f69b-40ba-80ec-347dcee30796","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:31.700Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d7d76a09-9d0e-4f2f-b188-25de9330a975","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:31.715Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d7d76a09-9d0e-4f2f-b188-25de9330a975","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:31.715Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"d7d76a09-9d0e-4f2f-b188-25de9330a975","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:31.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"890f2011-04a4-4104-92b3-088af2c1d636","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:31.789Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"890f2011-04a4-4104-92b3-088af2c1d636","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:31.789Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"890f2011-04a4-4104-92b3-088af2c1d636","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:31.789Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"890f2011-04a4-4104-92b3-088af2c1d636"}
{"level":"info","ts":"2026-06-11T10:21:31.789Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"890f2011-04a4-4104-92b3-088af2c1d636","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8"}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8"}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:33.686Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:21:33.687Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","podName":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:21:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:21:33.687Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:33.687Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"9b4be435-2e01-43b4-9e38-a86c9e1a26a8","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:34.790Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c4ee33a5-7d61-4241-8b09-86dbb0833877","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:34.790Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c4ee33a5-7d61-4241-8b09-86dbb0833877","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:34.790Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c4ee33a5-7d61-4241-8b09-86dbb0833877","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:34.790Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c4ee33a5-7d61-4241-8b09-86dbb0833877"}
{"level":"info","ts":"2026-06-11T10:21:34.790Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"c4ee33a5-7d61-4241-8b09-86dbb0833877","RayCluster":"fail-k8s-job-b6vx5","State":""}
{"level":"info","ts":"2026-06-11T10:21:36.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e"}
{"level":"info","ts":"2026-06-11T10:21:36.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e"}
{"level":"info","ts":"2026-06-11T10:21:36.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:36.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:36.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:36.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:36.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:36.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:36.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:36.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:36.123Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:21:36.123Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"2e690fad-9181-4bc6-9652-a40acb1be26e","name":"fail-k8s-job-b6vx5","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:21:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:21:36Z","stateTransitionTimes":{"ready":"2026-06-11T10:21:36Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.27","serviceIP":"10.133.0.27","podName":"fail-k8s-job-b6vx5-head-zn8b8","serviceName":"fail-k8s-job-b6vx5-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:21:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:21:36Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:21:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:21:11Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"56b49f07-c483-40f4-b70b-a89e1cec240b","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffbe0630-475a-4344-aefc-71c8c4a8cadf","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffbe0630-475a-4344-aefc-71c8c4a8cadf","pod":"fail-k8s-job-b6vx5-head-zn8b8","ip":"10.133.0.27"}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffbe0630-475a-4344-aefc-71c8c4a8cadf","pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","ip":"10.134.0.37"}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ffbe0630-475a-4344-aefc-71c8c4a8cadf","rayCluster":"fail-k8s-job-b6vx5","podIPs":["10.133.0.27","10.134.0.37"]}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.136Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.137Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166"}
{"level":"info","ts":"2026-06-11T10:21:36.137Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","head service name":"fail-k8s-job-b6vx5-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:36.137Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","head service URL":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:21:36.137Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166"}
{"level":"info","ts":"2026-06-11T10:21:36.137Z","logger":"controllers.RayJob","msg":"User-provided command is used","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","command":["ray","job","submit","--address","http://do-not-exist:8265","--","echo 123"]}
{"level":"info","ts":"2026-06-11T10:21:36.152Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-06-11T10:21:36.152Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.153Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-8gn4s","rayClusterName":"fail-k8s-job-b6vx5","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-8gn4s","rayClusterName":"fail-k8s-job-b6vx5","dashboardURL":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:21:36.153Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"9249cd65-310f-4d3f-908f-7a4a2f86a166","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:21:36.159Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"56b49f07-c483-40f4-b70b-a89e1cec240b","name":"fail-k8s-job-b6vx5-head"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","head service name":"fail-k8s-job-b6vx5-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","head service URL":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-8gn4s","rayClusterName":"fail-k8s-job-b6vx5","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-8gn4s","rayClusterName":"fail-k8s-job-b6vx5","dashboardURL":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:21:36.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:21:36.173Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:21:36.173Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63"}
{"level":"error","ts":"2026-06-11T10:21:36.173Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e01258e3-d564-453d-beca-600075138e63","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.173Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"01ae85f5-3215-413a-b526-8d60f15d544a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.173Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"01ae85f5-3215-413a-b526-8d60f15d544a","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.173Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"01ae85f5-3215-413a-b526-8d60f15d544a","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.173Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"01ae85f5-3215-413a-b526-8d60f15d544a"}
{"level":"info","ts":"2026-06-11T10:21:36.174Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"56b49f07-c483-40f4-b70b-a89e1cec240b","name":"fail-k8s-job-b6vx5-workers"}
{"level":"info","ts":"2026-06-11T10:21:36.174Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"56b49f07-c483-40f4-b70b-a89e1cec240b","cluster":"fail-k8s-job-b6vx5"}
{"level":"error","ts":"2026-06-11T10:21:36.182Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"01ae85f5-3215-413a-b526-8d60f15d544a","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.183Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"01ae85f5-3215-413a-b526-8d60f15d544a"}
{"level":"error","ts":"2026-06-11T10:21:36.183Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"01ae85f5-3215-413a-b526-8d60f15d544a","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.183Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"acb8b76c-1aae-46d7-9921-6c9bf04a8ff5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.183Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"acb8b76c-1aae-46d7-9921-6c9bf04a8ff5","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.183Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"acb8b76c-1aae-46d7-9921-6c9bf04a8ff5","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.183Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"acb8b76c-1aae-46d7-9921-6c9bf04a8ff5"}
{"level":"error","ts":"2026-06-11T10:21:36.232Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"acb8b76c-1aae-46d7-9921-6c9bf04a8ff5","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.232Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"acb8b76c-1aae-46d7-9921-6c9bf04a8ff5"}
{"level":"error","ts":"2026-06-11T10:21:36.232Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"acb8b76c-1aae-46d7-9921-6c9bf04a8ff5","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.232Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"753882a7-306d-4b1a-a740-eeef6c371d56","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"753882a7-306d-4b1a-a740-eeef6c371d56","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"753882a7-306d-4b1a-a740-eeef6c371d56","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"753882a7-306d-4b1a-a740-eeef6c371d56"}
{"level":"error","ts":"2026-06-11T10:21:36.237Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"753882a7-306d-4b1a-a740-eeef6c371d56","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.237Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"753882a7-306d-4b1a-a740-eeef6c371d56"}
{"level":"error","ts":"2026-06-11T10:21:36.237Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"753882a7-306d-4b1a-a740-eeef6c371d56","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"61334976-b587-45b1-96dd-b40d8db477a4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"61334976-b587-45b1-96dd-b40d8db477a4","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"61334976-b587-45b1-96dd-b40d8db477a4","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"61334976-b587-45b1-96dd-b40d8db477a4"}
{"level":"error","ts":"2026-06-11T10:21:36.258Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"61334976-b587-45b1-96dd-b40d8db477a4","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.258Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"61334976-b587-45b1-96dd-b40d8db477a4"}
{"level":"error","ts":"2026-06-11T10:21:36.258Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"61334976-b587-45b1-96dd-b40d8db477a4","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"46492454-cbff-454e-8f07-74dcfdeead9c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"46492454-cbff-454e-8f07-74dcfdeead9c","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"46492454-cbff-454e-8f07-74dcfdeead9c","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"46492454-cbff-454e-8f07-74dcfdeead9c"}
{"level":"error","ts":"2026-06-11T10:21:36.343Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"46492454-cbff-454e-8f07-74dcfdeead9c","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.343Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"46492454-cbff-454e-8f07-74dcfdeead9c"}
{"level":"error","ts":"2026-06-11T10:21:36.343Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"46492454-cbff-454e-8f07-74dcfdeead9c","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.504Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"a34b4d87-d42f-4853-ae47-1e41606ba0b1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.504Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"a34b4d87-d42f-4853-ae47-1e41606ba0b1","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.504Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"a34b4d87-d42f-4853-ae47-1e41606ba0b1","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.504Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"a34b4d87-d42f-4853-ae47-1e41606ba0b1"}
{"level":"error","ts":"2026-06-11T10:21:36.510Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"a34b4d87-d42f-4853-ae47-1e41606ba0b1","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.510Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"a34b4d87-d42f-4853-ae47-1e41606ba0b1"}
{"level":"error","ts":"2026-06-11T10:21:36.510Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"a34b4d87-d42f-4853-ae47-1e41606ba0b1","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b9afd3b8-7bfc-4a1b-a1ea-14ed4aae29bb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:36.830Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b9afd3b8-7bfc-4a1b-a1ea-14ed4aae29bb","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:36.830Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b9afd3b8-7bfc-4a1b-a1ea-14ed4aae29bb","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:36.830Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b9afd3b8-7bfc-4a1b-a1ea-14ed4aae29bb"}
{"level":"error","ts":"2026-06-11T10:21:36.836Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b9afd3b8-7bfc-4a1b-a1ea-14ed4aae29bb","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:36.836Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b9afd3b8-7bfc-4a1b-a1ea-14ed4aae29bb"}
{"level":"error","ts":"2026-06-11T10:21:36.836Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b9afd3b8-7bfc-4a1b-a1ea-14ed4aae29bb","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:37.178Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b0325a7b-b0a1-4a87-a414-8eec35cccf60","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:37.178Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b0325a7b-b0a1-4a87-a414-8eec35cccf60","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:37.178Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b0325a7b-b0a1-4a87-a414-8eec35cccf60","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:37.178Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b0325a7b-b0a1-4a87-a414-8eec35cccf60"}
{"level":"error","ts":"2026-06-11T10:21:37.184Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b0325a7b-b0a1-4a87-a414-8eec35cccf60","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:37.184Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b0325a7b-b0a1-4a87-a414-8eec35cccf60"}
{"level":"error","ts":"2026-06-11T10:21:37.184Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b0325a7b-b0a1-4a87-a414-8eec35cccf60","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:37.476Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2bb64ec0-6d73-4e0f-9f83-c7cdff793223","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:37.476Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2bb64ec0-6d73-4e0f-9f83-c7cdff793223","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:37.476Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2bb64ec0-6d73-4e0f-9f83-c7cdff793223","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:37.476Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2bb64ec0-6d73-4e0f-9f83-c7cdff793223"}
{"level":"error","ts":"2026-06-11T10:21:37.482Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2bb64ec0-6d73-4e0f-9f83-c7cdff793223","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:37.482Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2bb64ec0-6d73-4e0f-9f83-c7cdff793223"}
{"level":"error","ts":"2026-06-11T10:21:37.482Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2bb64ec0-6d73-4e0f-9f83-c7cdff793223","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295"}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295"}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","1 head service found":"fail-k8s-job-b6vx5-head-svc"}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","Found 1 head Pod":"fail-k8s-job-b6vx5-head-zn8b8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","head Pod":"fail-k8s-job-b6vx5-head-zn8b8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-b6vx5-head-zn8b8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","worker Pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-b6vx5-small-group-worker-hhh7t. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:21:38.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:21:38.138Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:21:38.138Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"5dbf29f8-0666-4cf8-9c0d-771ac36b9295","seconds":300}
{"level":"info","ts":"2026-06-11T10:21:40.043Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"eccba07f-3f80-427d-a298-c989b7196316","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:40.043Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"eccba07f-3f80-427d-a298-c989b7196316","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:40.043Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"eccba07f-3f80-427d-a298-c989b7196316","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:40.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"eccba07f-3f80-427d-a298-c989b7196316"}
{"level":"error","ts":"2026-06-11T10:21:40.048Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"eccba07f-3f80-427d-a298-c989b7196316","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:40.048Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"eccba07f-3f80-427d-a298-c989b7196316"}
{"level":"error","ts":"2026-06-11T10:21:40.048Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"eccba07f-3f80-427d-a298-c989b7196316","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:41.134Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"b3eebc3a-f139-461b-8c0b-d657f45c4f2a","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:41.134Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"b3eebc3a-f139-461b-8c0b-d657f45c4f2a","pod":"fail-k8s-job-b6vx5-head-zn8b8","ip":"10.133.0.27"}
{"level":"info","ts":"2026-06-11T10:21:41.134Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"b3eebc3a-f139-461b-8c0b-d657f45c4f2a","pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","ip":"10.134.0.37"}
{"level":"info","ts":"2026-06-11T10:21:41.134Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"b3eebc3a-f139-461b-8c0b-d657f45c4f2a","rayCluster":"fail-k8s-job-b6vx5","podIPs":["10.133.0.27","10.134.0.37"]}
{"level":"info","ts":"2026-06-11T10:21:42.177Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e23693d7-d4e1-48e7-94e6-70137eb4dae1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:42.177Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e23693d7-d4e1-48e7-94e6-70137eb4dae1","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:42.177Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e23693d7-d4e1-48e7-94e6-70137eb4dae1","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:42.177Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e23693d7-d4e1-48e7-94e6-70137eb4dae1"}
{"level":"error","ts":"2026-06-11T10:21:42.183Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e23693d7-d4e1-48e7-94e6-70137eb4dae1","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:42.183Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e23693d7-d4e1-48e7-94e6-70137eb4dae1"}
{"level":"error","ts":"2026-06-11T10:21:42.183Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"e23693d7-d4e1-48e7-94e6-70137eb4dae1","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:42.646Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a70b617b-d829-4a00-9d37-e00a6bbca892","namespace":"test-ns-c6srn","clusterName":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:42.646Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-88qlb","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-88qlb","reconcileID":"a70b617b-d829-4a00-9d37-e00a6bbca892","namespace":"test-ns-c6srn","clusterName":"fail-88qlb"}
{"level":"info","ts":"2026-06-11T10:21:43.186Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ae3b82b0-7601-48da-9077-239464b72c65","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:43.186Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ae3b82b0-7601-48da-9077-239464b72c65","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:43.186Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ae3b82b0-7601-48da-9077-239464b72c65","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:43.187Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ae3b82b0-7601-48da-9077-239464b72c65"}
{"level":"error","ts":"2026-06-11T10:21:43.192Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ae3b82b0-7601-48da-9077-239464b72c65","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:43.192Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ae3b82b0-7601-48da-9077-239464b72c65"}
{"level":"error","ts":"2026-06-11T10:21:43.192Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"ae3b82b0-7601-48da-9077-239464b72c65","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:43.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7e503046-f102-4a28-9aa3-87b771b26bdb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:43.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7e503046-f102-4a28-9aa3-87b771b26bdb","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:43.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7e503046-f102-4a28-9aa3-87b771b26bdb","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:43.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7e503046-f102-4a28-9aa3-87b771b26bdb"}
{"level":"error","ts":"2026-06-11T10:21:43.210Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7e503046-f102-4a28-9aa3-87b771b26bdb","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:43.210Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7e503046-f102-4a28-9aa3-87b771b26bdb"}
{"level":"error","ts":"2026-06-11T10:21:43.210Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7e503046-f102-4a28-9aa3-87b771b26bdb","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:45.169Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"baba70c8-fa7b-43c5-ac3d-d83a3d8f0ea7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:45.169Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"baba70c8-fa7b-43c5-ac3d-d83a3d8f0ea7","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:45.169Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"baba70c8-fa7b-43c5-ac3d-d83a3d8f0ea7","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:45.169Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"baba70c8-fa7b-43c5-ac3d-d83a3d8f0ea7"}
{"level":"error","ts":"2026-06-11T10:21:45.175Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"baba70c8-fa7b-43c5-ac3d-d83a3d8f0ea7","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:45.175Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"baba70c8-fa7b-43c5-ac3d-d83a3d8f0ea7"}
{"level":"error","ts":"2026-06-11T10:21:45.175Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"baba70c8-fa7b-43c5-ac3d-d83a3d8f0ea7","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:50.034Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2afd69a3-4f37-466e-9305-7c7a8e41bdb6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:50.034Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2afd69a3-4f37-466e-9305-7c7a8e41bdb6","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:50.034Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2afd69a3-4f37-466e-9305-7c7a8e41bdb6","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:50.034Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2afd69a3-4f37-466e-9305-7c7a8e41bdb6"}
{"level":"error","ts":"2026-06-11T10:21:50.040Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2afd69a3-4f37-466e-9305-7c7a8e41bdb6","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:50.040Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2afd69a3-4f37-466e-9305-7c7a8e41bdb6"}
{"level":"error","ts":"2026-06-11T10:21:50.040Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"2afd69a3-4f37-466e-9305-7c7a8e41bdb6","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:52.249Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d747625f-b8a7-4b34-8622-fe9e0e5285f3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:52.250Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d747625f-b8a7-4b34-8622-fe9e0e5285f3","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:52.250Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d747625f-b8a7-4b34-8622-fe9e0e5285f3","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:52.250Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d747625f-b8a7-4b34-8622-fe9e0e5285f3"}
{"level":"error","ts":"2026-06-11T10:21:52.255Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d747625f-b8a7-4b34-8622-fe9e0e5285f3","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:52.255Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d747625f-b8a7-4b34-8622-fe9e0e5285f3"}
{"level":"error","ts":"2026-06-11T10:21:52.255Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d747625f-b8a7-4b34-8622-fe9e0e5285f3","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:55.252Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b55070ed-b5bf-4e27-98d6-4ee86c6d8345","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:55.252Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b55070ed-b5bf-4e27-98d6-4ee86c6d8345","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:55.252Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b55070ed-b5bf-4e27-98d6-4ee86c6d8345","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:55.252Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b55070ed-b5bf-4e27-98d6-4ee86c6d8345"}
{"level":"error","ts":"2026-06-11T10:21:55.258Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b55070ed-b5bf-4e27-98d6-4ee86c6d8345","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:55.258Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b55070ed-b5bf-4e27-98d6-4ee86c6d8345"}
{"level":"error","ts":"2026-06-11T10:21:55.258Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"b55070ed-b5bf-4e27-98d6-4ee86c6d8345","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:56.263Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"5d3edd04-00c9-4fd9-a799-d8005665b195","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:56.263Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"5d3edd04-00c9-4fd9-a799-d8005665b195","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:56.263Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"5d3edd04-00c9-4fd9-a799-d8005665b195","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:56.263Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"5d3edd04-00c9-4fd9-a799-d8005665b195"}
{"level":"error","ts":"2026-06-11T10:21:56.269Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"5d3edd04-00c9-4fd9-a799-d8005665b195","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:56.269Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"5d3edd04-00c9-4fd9-a799-d8005665b195"}
{"level":"error","ts":"2026-06-11T10:21:56.269Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"5d3edd04-00c9-4fd9-a799-d8005665b195","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:56.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42907153-9138-470e-86c1-585fd4f36b2e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:21:56.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42907153-9138-470e-86c1-585fd4f36b2e","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:21:56.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42907153-9138-470e-86c1-585fd4f36b2e","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:21:56.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42907153-9138-470e-86c1-585fd4f36b2e"}
{"level":"error","ts":"2026-06-11T10:21:56.302Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42907153-9138-470e-86c1-585fd4f36b2e","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:21:56.302Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42907153-9138-470e-86c1-585fd4f36b2e"}
{"level":"error","ts":"2026-06-11T10:21:56.302Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42907153-9138-470e-86c1-585fd4f36b2e","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:14.025Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"93b9917a-e61a-43b7-865a-20d6d3e03d9a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:14.025Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"93b9917a-e61a-43b7-865a-20d6d3e03d9a","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:14.025Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"93b9917a-e61a-43b7-865a-20d6d3e03d9a","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:14.025Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"93b9917a-e61a-43b7-865a-20d6d3e03d9a"}
{"level":"error","ts":"2026-06-11T10:22:14.034Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"93b9917a-e61a-43b7-865a-20d6d3e03d9a","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:14.034Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"93b9917a-e61a-43b7-865a-20d6d3e03d9a"}
{"level":"error","ts":"2026-06-11T10:22:14.034Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"93b9917a-e61a-43b7-865a-20d6d3e03d9a","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:16.348Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cc4851bc-ad8d-4da9-8237-94fac260f866","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:16.348Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cc4851bc-ad8d-4da9-8237-94fac260f866","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:16.348Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cc4851bc-ad8d-4da9-8237-94fac260f866","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:16.348Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cc4851bc-ad8d-4da9-8237-94fac260f866"}
{"level":"error","ts":"2026-06-11T10:22:16.354Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cc4851bc-ad8d-4da9-8237-94fac260f866","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:16.354Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cc4851bc-ad8d-4da9-8237-94fac260f866"}
{"level":"error","ts":"2026-06-11T10:22:16.354Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cc4851bc-ad8d-4da9-8237-94fac260f866","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:16.587Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6a4e257f-8e3a-433a-983a-ae5076e454de","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:22:16.587Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6a4e257f-8e3a-433a-983a-ae5076e454de","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:22:16.587Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6a4e257f-8e3a-433a-983a-ae5076e454de","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:22:16.587Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"6a4e257f-8e3a-433a-983a-ae5076e454de","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9"}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9"}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","Found 1 head Pod":"raycluster-head-t5q9r","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","head Pod":"raycluster-head-t5q9r","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-t5q9r. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","worker Pod":"raycluster-small-group-worker-n98tf","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-n98tf. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:17.623Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"reconcileID":"fbf20e67-797a-4b43-b3d2-2b800968b8a9","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:19.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8048eb3-0fa5-4eaa-8f0c-bc8c5e8eb8ea","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:19.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8048eb3-0fa5-4eaa-8f0c-bc8c5e8eb8ea","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:19.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8048eb3-0fa5-4eaa-8f0c-bc8c5e8eb8ea","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:19.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8048eb3-0fa5-4eaa-8f0c-bc8c5e8eb8ea"}
{"level":"error","ts":"2026-06-11T10:22:19.361Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8048eb3-0fa5-4eaa-8f0c-bc8c5e8eb8ea","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:19.361Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8048eb3-0fa5-4eaa-8f0c-bc8c5e8eb8ea"}
{"level":"error","ts":"2026-06-11T10:22:19.361Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"d8048eb3-0fa5-4eaa-8f0c-bc8c5e8eb8ea","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.365Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"421ecbd3-ae97-4f41-8912-dc80501e4cf8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:20.365Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"421ecbd3-ae97-4f41-8912-dc80501e4cf8","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.365Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"421ecbd3-ae97-4f41-8912-dc80501e4cf8","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.365Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"421ecbd3-ae97-4f41-8912-dc80501e4cf8"}
{"level":"error","ts":"2026-06-11T10:22:20.371Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"421ecbd3-ae97-4f41-8912-dc80501e4cf8","JobId":"fail-k8s-job-8gn4s","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.371Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"421ecbd3-ae97-4f41-8912-dc80501e4cf8"}
{"level":"error","ts":"2026-06-11T10:22:20.371Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"421ecbd3-ae97-4f41-8912-dc80501e4cf8","error":"Job fail-k8s-job-8gn4s does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.388Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"8f4a5bbc-49a4-4d07-8de3-acad52dbe40e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:20.388Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"8f4a5bbc-49a4-4d07-8de3-acad52dbe40e","Submitter K8s Job":"fail-k8s-job","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-06-11T10:22:20.388Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"8f4a5bbc-49a4-4d07-8de3-acad52dbe40e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-8gn4s","rayClusterName":"fail-k8s-job-b6vx5","dashboardURL":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-8gn4s","rayClusterName":"fail-k8s-job-b6vx5","dashboardURL":"fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-06-11T10:21:10Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:22:20.388Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"8f4a5bbc-49a4-4d07-8de3-acad52dbe40e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-11T10:22:20.404Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"91117692-e5e0-4cde-b594-f3892ba93f3f","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:20.404Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"91117692-e5e0-4cde-b594-f3892ba93f3f","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:22:20 +0000 UTC","Now":"2026-06-11T10:22:20.404Z","ShutdownTime":"2026-06-11T10:22:20.000Z"}
{"level":"info","ts":"2026-06-11T10:22:20.404Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"91117692-e5e0-4cde-b594-f3892ba93f3f","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:22:20.416Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"91117692-e5e0-4cde-b594-f3892ba93f3f","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.416Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"91117692-e5e0-4cde-b594-f3892ba93f3f","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:22:20.416Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"91117692-e5e0-4cde-b594-f3892ba93f3f","RayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"reconcileID":"1acc12f3-b7cc-4829-957b-5a97706d87c9"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"c7d37a83-2c0a-423f-a0b7-2c35c638b119"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3df96eff-3277-4ae2-a03c-18f4b6590433","rayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7f855883-7240-49c7-bbd1-573c351cab1d","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7f855883-7240-49c7-bbd1-573c351cab1d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:22:20 +0000 UTC","Now":"2026-06-11T10:22:20.417Z","ShutdownTime":"2026-06-11T10:22:20.000Z"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7f855883-7240-49c7-bbd1-573c351cab1d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3df96eff-3277-4ae2-a03c-18f4b6590433","pod":"fail-k8s-job-b6vx5-head-zn8b8","ip":"10.133.0.27"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3df96eff-3277-4ae2-a03c-18f4b6590433","pod":"fail-k8s-job-b6vx5-small-group-worker-hhh7t","ip":"10.134.0.37"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7f855883-7240-49c7-bbd1-573c351cab1d","RayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7f855883-7240-49c7-bbd1-573c351cab1d","isClusterDeleted":false}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"7f855883-7240-49c7-bbd1-573c351cab1d","RayCluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.417Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3df96eff-3277-4ae2-a03c-18f4b6590433","rayCluster":"fail-k8s-job-b6vx5","podIPs":["10.133.0.27","10.134.0.37"]}
{"level":"info","ts":"2026-06-11T10:22:20.424Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","name":"kuberay-gateway-access","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:20.432Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","name":"test-ns-c6srn-fail-k8s-job-b6vx5","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:22:20.442Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","configMap":"kube-rbac-proxy-config-fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.451Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","serviceAccount":"fail-k8s-job-b6vx5-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:22:20.459Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.459Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:20.470Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"63aaa7a2-a6c4-4550-8747-893929c994a4","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.470Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.470Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.470Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55","cluster":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.470Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:22:20.470Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"ac995916-34b4-4738-b965-8195cf8bbd88"}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"db1f70a9-ed52-467f-9e50-ffa3ff6dc3e2","namespace":"test-ns-c6srn","clusterName":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4aa6edc3-e3a2-4795-b7c4-3a1f642be5a2","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4aa6edc3-e3a2-4795-b7c4-3a1f642be5a2","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:22:20 +0000 UTC","Now":"2026-06-11T10:22:20.471Z","ShutdownTime":"2026-06-11T10:22:20.000Z"}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4aa6edc3-e3a2-4795-b7c4-3a1f642be5a2","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4aa6edc3-e3a2-4795-b7c4-3a1f642be5a2","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4aa6edc3-e3a2-4795-b7c4-3a1f642be5a2","isClusterDeleted":true}
{"level":"info","ts":"2026-06-11T10:22:20.471Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"4aa6edc3-e3a2-4795-b7c4-3a1f642be5a2","RayCluster":"fail-k8s-job-b6vx5"}
{"level":"error","ts":"2026-06-11T10:22:20.475Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55","error":"rayclusters.ray.io \"fail-k8s-job-b6vx5\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.475Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55"}
{"level":"error","ts":"2026-06-11T10:22:20.475Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"8603f38e-595f-4630-94a4-308cc9525a55","error":"rayclusters.ray.io \"fail-k8s-job-b6vx5\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.475Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"e2dc7690-6411-4f96-a071-970c2bd7fd4a","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.475Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"e2dc7690-6411-4f96-a071-970c2bd7fd4a"}
{"level":"info","ts":"2026-06-11T10:22:20.475Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"e2dc7690-6411-4f96-a071-970c2bd7fd4a","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:22:20.475Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"e2dc7690-6411-4f96-a071-970c2bd7fd4a","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:20.480Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"629c9641-1031-45e7-b656-da00df1bff22","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.480Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"629c9641-1031-45e7-b656-da00df1bff22"}
{"level":"info","ts":"2026-06-11T10:22:20.480Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"629c9641-1031-45e7-b656-da00df1bff22","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:22:20.480Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"629c9641-1031-45e7-b656-da00df1bff22","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:20.487Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"5a47fba9-f663-443d-86cc-0027a9f9821b"}
{"level":"error","ts":"2026-06-11T10:22:20.492Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"db1f70a9-ed52-467f-9e50-ffa3ff6dc3e2","issuer":"ray-selfsigned-issuer-fail-k8s-job-b6vx5","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-b6vx5\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.507Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"00eb6749-20ff-4d0c-903c-088c8219ac0b","namespacedName":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:20.507Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"00eb6749-20ff-4d0c-903c-088c8219ac0b"}
{"level":"info","ts":"2026-06-11T10:22:20.507Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"00eb6749-20ff-4d0c-903c-088c8219ac0b","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:22:20.507Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"00eb6749-20ff-4d0c-903c-088c8219ac0b","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:20.511Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"babaa695-3fe5-4800-93dc-e339736fd5ef"}
{"level":"error","ts":"2026-06-11T10:22:20.525Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"db1f70a9-ed52-467f-9e50-ffa3ff6dc3e2","certificate":"ray-ca-certificate-fail-k8s-job-b6vx5","error":"certificates.cert-manager.io \"ray-ca-certificate-fail-k8s-job-b6vx5\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.575Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"db1f70a9-ed52-467f-9e50-ffa3ff6dc3e2","errorCount":2}
{"level":"info","ts":"2026-06-11T10:22:20.575Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"db1f70a9-ed52-467f-9e50-ffa3ff6dc3e2"}
{"level":"error","ts":"2026-06-11T10:22:20.575Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"db1f70a9-ed52-467f-9e50-ffa3ff6dc3e2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-b6vx5\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:20.581Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3f89886e-2253-42ba-93ca-15eeabcca619","namespace":"test-ns-c6srn","clusterName":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:20.581Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-b6vx5","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"fail-k8s-job-b6vx5","reconcileID":"3f89886e-2253-42ba-93ca-15eeabcca619","namespace":"test-ns-c6srn","clusterName":"fail-k8s-job-b6vx5"}
{"level":"info","ts":"2026-06-11T10:22:21.229Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"30018537-4215-456f-9277-3f2466d3efcd","DeletionTimestamp":"2026-06-11 10:22:21 +0000 UTC"}
{"level":"error","ts":"2026-06-11T10:22:21.229Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"30018537-4215-456f-9277-3f2466d3efcd","error":"RayCluster.ray.io \"fail-k8s-job-b6vx5\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:21.229Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"30018537-4215-456f-9277-3f2466d3efcd","rayJob":"fail-k8s-job-8gn4s"}
{"level":"error","ts":"2026-06-11T10:22:23.229Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"30018537-4215-456f-9277-3f2466d3efcd","error":"Post \"http://fail-k8s-job-b6vx5-head-svc.test-ns-c6srn.svc.cluster.local:8265/api/jobs/fail-k8s-job-8gn4s/stop\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.229Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"30018537-4215-456f-9277-3f2466d3efcd","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:22:23.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"39d11c71-c964-4c52-8411-5b2346176379","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:23.248Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"39d11c71-c964-4c52-8411-5b2346176379","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:22:23.269Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"39d11c71-c964-4c52-8411-5b2346176379"}
{"level":"info","ts":"2026-06-11T10:22:23.269Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"39d11c71-c964-4c52-8411-5b2346176379","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:22:23.269Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"39d11c71-c964-4c52-8411-5b2346176379","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:22:23.269Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"39d11c71-c964-4c52-8411-5b2346176379","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:22:23.288Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"cfd03593-1388-4403-9034-40eb5b4fb755"}
{"level":"info","ts":"2026-06-11T10:22:23.289Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb1d627c-3baa-4a94-8857-420b9e5fdd5a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:23.289Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb1d627c-3baa-4a94-8857-420b9e5fdd5a","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.289Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb1d627c-3baa-4a94-8857-420b9e5fdd5a","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.289Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb1d627c-3baa-4a94-8857-420b9e5fdd5a","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.297Z","logger":"raycluster-resource","msg":"default","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.297Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-cn5qj","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb1d627c-3baa-4a94-8857-420b9e5fdd5a","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb1d627c-3baa-4a94-8857-420b9e5fdd5a"}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb1d627c-3baa-4a94-8857-420b9e5fdd5a","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"11bbeaa1-a159-4602-8d16-01886f0e3fb3"}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9719b725-0fcf-4297-a252-e5014d63cf5d","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9719b725-0fcf-4297-a252-e5014d63cf5d","mode":"IntegratedOAuth","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9719b725-0fcf-4297-a252-e5014d63cf5d","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9719b725-0fcf-4297-a252-e5014d63cf5d","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"34a63652-3eeb-4b59-bf24-ce9eaa0229a5","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53"}
{"level":"info","ts":"2026-06-11T10:22:23.325Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"043bbeb2-532d-45a2-b9db-eb70001fa949","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"043bbeb2-532d-45a2-b9db-eb70001fa949","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"043bbeb2-532d-45a2-b9db-eb70001fa949","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"043bbeb2-532d-45a2-b9db-eb70001fa949"}
{"level":"info","ts":"2026-06-11T10:22:23.326Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"043bbeb2-532d-45a2-b9db-eb70001fa949","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:23.350Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","name":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:23.350Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:23.350Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.350Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:23.350Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.350Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.350Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","serviceSelectors":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/identifier":"stop-cn5qj-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","error":"unable to find head service. cluster name stop-cn5qj, filter labels [test-ns-c6srn map[ray.io/cluster:stop-cn5qj ray.io/identifier:stop-cn5qj-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53"}
{"level":"error","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"23d953b5-994b-4a49-8797-055cc220ae53","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164"}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164"}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:22:23.351Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","name":"stop-cn5qj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:22:23.353Z","logger":"raycluster-resource","msg":"default","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.353Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-cn5qj","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:23.362Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"34a63652-3eeb-4b59-bf24-ce9eaa0229a5","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.362Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"a3ea2677-4ef6-4911-ae39-0cabd16c196e","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164"}
{"level":"error","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"3957ae02-8c85-4716-9414-8dea9e123164","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e2144f96-e780-461e-b1a9-545b6f081377","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e2144f96-e780-461e-b1a9-545b6f081377","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e2144f96-e780-461e-b1a9-545b6f081377","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.366Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.367Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.367Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e2144f96-e780-461e-b1a9-545b6f081377"}
{"level":"info","ts":"2026-06-11T10:22:23.367Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e2144f96-e780-461e-b1a9-545b6f081377","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:23.367Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56"}
{"level":"error","ts":"2026-06-11T10:22:23.367Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"560fd17a-414e-4a9e-b69e-4a9e2321ea56","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.369Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a"}
{"level":"info","ts":"2026-06-11T10:22:23.369Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:22:23.369Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.376Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"a3ea2677-4ef6-4911-ae39-0cabd16c196e","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:22:23.376Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3"}
{"level":"info","ts":"2026-06-11T10:22:23.376Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3"}
{"level":"info","ts":"2026-06-11T10:22:23.376Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:23.376Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:23.376Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.376Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:23.376Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.377Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.377Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.377Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3"}
{"level":"error","ts":"2026-06-11T10:22:23.377Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"dca98d62-a9d3-4254-875b-7773afc554c3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-c6srn"},"reconcileID":"42dc1e80-5717-41c5-a104-772a0f9363b1"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"raycluster-resource","msg":"default","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-cn5qj","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8"}
{"level":"error","ts":"2026-06-11T10:22:23.427Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"242dd7d3-2dae-4bce-b266-c71fb58504e8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.427Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj"}
{"level":"error","ts":"2026-06-11T10:22:23.430Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9719b725-0fcf-4297-a252-e5014d63cf5d","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-cn5qj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.431Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9719b725-0fcf-4297-a252-e5014d63cf5d"}
{"level":"error","ts":"2026-06-11T10:22:23.431Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9719b725-0fcf-4297-a252-e5014d63cf5d","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-cn5qj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.431Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.431Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","mode":"IntegratedOAuth","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.431Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.431Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.432Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"a3ea2677-4ef6-4911-ae39-0cabd16c196e","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:22:23.432Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"a3ea2677-4ef6-4911-ae39-0cabd16c196e","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.432Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"01ca954d-235b-42af-a57b-480c05e46a6d","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.437Z","logger":"raycluster-resource","msg":"default","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.437Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-cn5qj","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:22:23.440Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a"}
{"level":"info","ts":"2026-06-11T10:22:23.440Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:22:23.451Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"01ca954d-235b-42af-a57b-480c05e46a6d","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a"}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844"}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844"}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844"}
{"level":"error","ts":"2026-06-11T10:22:23.527Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"01201e4a-309c-4521-a92a-7084e7879844","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a","rayCluster":"stop-cn5qj","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"1e5c0126-1f47-4555-9610-318facd8127a"}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"36ce4d43-ff04-4915-9ec0-513e1c5c3fb3","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"36ce4d43-ff04-4915-9ec0-513e1c5c3fb3"}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4ab79492-554b-4622-8503-367ac9618f8f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4ab79492-554b-4622-8503-367ac9618f8f","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4ab79492-554b-4622-8503-367ac9618f8f","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4ab79492-554b-4622-8503-367ac9618f8f"}
{"level":"info","ts":"2026-06-11T10:22:23.528Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4ab79492-554b-4622-8503-367ac9618f8f","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:23.533Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"01ca954d-235b-42af-a57b-480c05e46a6d","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:22:23.533Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"01ca954d-235b-42af-a57b-480c05e46a6d","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.533Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e1ffef29-5245-4e9d-9f45-7b460ef5b4e0","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.538Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","name":"stop-cn5qj-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:22:23.545Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","name":"kuberay-gateway-access","namespace":"test-ns-c6srn","operation":"created"}
{"level":"info","ts":"2026-06-11T10:22:23.551Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e1ffef29-5245-4e9d-9f45-7b460ef5b4e0","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:22:23.553Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","name":"test-ns-c6srn-stop-cn5qj","operation":"created"}
{"level":"info","ts":"2026-06-11T10:22:23.561Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","name":"kube-rbac-proxy-config-stop-cn5qj","operation":"created"}
{"level":"info","ts":"2026-06-11T10:22:23.627Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e1ffef29-5245-4e9d-9f45-7b460ef5b4e0","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:22:23.627Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e1ffef29-5245-4e9d-9f45-7b460ef5b4e0","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b52464ab-5126-4f76-a5e6-801a17ca499a","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b4cf1f42-8842-4508-96fb-7974a41261ea","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b4cf1f42-8842-4508-96fb-7974a41261ea","mode":"IntegratedOAuth","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b4cf1f42-8842-4508-96fb-7974a41261ea","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"344c9399-1265-4468-a3ef-7fa57ff9e0d5","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4b933d2c-faeb-406a-a094-8ae6211ff67a","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"344c9399-1265-4468-a3ef-7fa57ff9e0d5"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"cf6f793a-1a9d-4a01-a6d8-4a816887f0a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"cf6f793a-1a9d-4a01-a6d8-4a816887f0a8","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"cf6f793a-1a9d-4a01-a6d8-4a816887f0a8","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"cf6f793a-1a9d-4a01-a6d8-4a816887f0a8"}
{"level":"info","ts":"2026-06-11T10:22:23.628Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"cf6f793a-1a9d-4a01-a6d8-4a816887f0a8","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:23.640Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b4cf1f42-8842-4508-96fb-7974a41261ea","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.648Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4b933d2c-faeb-406a-a094-8ae6211ff67a","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:22:23.662Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4b933d2c-faeb-406a-a094-8ae6211ff67a","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:22:23.662Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4b933d2c-faeb-406a-a094-8ae6211ff67a","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43"}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43"}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43"}
{"level":"error","ts":"2026-06-11T10:22:23.688Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"033f27be-c885-4626-9ca0-38390a815c43","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44"}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44"}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:24.008Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:24.009Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44"}
{"level":"error","ts":"2026-06-11T10:22:24.009Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"648cf9c2-f7a0-4432-9111-c4396884dd44","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:24.649Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297"}
{"level":"info","ts":"2026-06-11T10:22:24.649Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297"}
{"level":"info","ts":"2026-06-11T10:22:24.649Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:24.649Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","error":"head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"error","ts":"2026-06-11T10:22:24.649Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:24.650Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found"}
{"level":"info","ts":"2026-06-11T10:22:24.650Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:24.650Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:24.650Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","filter labels":["test-ns-c6srn",{"ray.io/cluster":"stop-cn5qj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:22:24.650Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297"}
{"level":"error","ts":"2026-06-11T10:22:24.650Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"aca8a669-84e1-42a5-a52d-573f6e3cf297","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-cn5qj not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","mode":"IntegratedOAuth","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","cluster":"stop-cn5qj","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","cluster":"stop-cn5qj","authType":"IntegratedOAuth","serviceAccount":"stop-cn5qj-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-cn5qj","ray.io/group":"headgroup","ray.io/identifier":"stop-cn5qj-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:22:25.930Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","enabled":true}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","name":"stop-cn5qj-head-9ljzg"}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e"}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","nodeType":"worker","rayStartParams":{"address":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","rayStartCmd":"ray start  --address=stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:22:25.951Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","enabled":true}
{"level":"info","ts":"2026-06-11T10:22:25.970Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","name":"stop-cn5qj-small-group-worker-xvwbb"}
{"level":"info","ts":"2026-06-11T10:22:25.970Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:25.970Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:25.971Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-cn5qj-head-svc"},"newHead":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"}}
{"level":"info","ts":"2026-06-11T10:22:25.971Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9831000e-e448-4d53-9249-8685d445154e","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:22:25.987Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"2435d1d9-424e-4c42-9a78-1c41ca6c92d3","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:25.987Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"2435d1d9-424e-4c42-9a78-1c41ca6c92d3","rayCluster":"stop-cn5qj","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:22:25.987Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c4c37c44-c53e-4a99-b2d6-142bd205f72a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:25.987Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c4c37c44-c53e-4a99-b2d6-142bd205f72a","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:25.987Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"64af0e3f-9ceb-41da-8fc4-1e8d17d60d81","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:25.987Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c4c37c44-c53e-4a99-b2d6-142bd205f72a","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c4c37c44-c53e-4a99-b2d6-142bd205f72a"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c4c37c44-c53e-4a99-b2d6-142bd205f72a","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"ea13103c-ae56-45bf-996d-5b1868fae8fc","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:25.988Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8b527eee-2927-427c-92ba-46115a61f89b","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:22:26.005Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"736285d1-a45b-4509-a919-b846c7f479a6","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:22:26.027Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"64af0e3f-9ceb-41da-8fc4-1e8d17d60d81","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:22:26.053Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"64af0e3f-9ceb-41da-8fc4-1e8d17d60d81","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:22:26.053Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"64af0e3f-9ceb-41da-8fc4-1e8d17d60d81","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ee3d7d35-5a78-45af-8a4d-dd6d1377f1c8","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e0bbd342-e10f-46fd-830f-5b961ebc308d","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ee3d7d35-5a78-45af-8a4d-dd6d1377f1c8","rayCluster":"stop-cn5qj","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:22:26.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4b65d072-e504-42a3-aa25-5c84e36f1735","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:26.061Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4b65d072-e504-42a3-aa25-5c84e36f1735","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.061Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4b65d072-e504-42a3-aa25-5c84e36f1735","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:26.061Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4b65d072-e504-42a3-aa25-5c84e36f1735"}
{"level":"info","ts":"2026-06-11T10:22:26.061Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4b65d072-e504-42a3-aa25-5c84e36f1735","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:26.100Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e0bbd342-e10f-46fd-830f-5b961ebc308d","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:22:26.128Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e0bbd342-e10f-46fd-830f-5b961ebc308d","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:22:26.128Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"e0bbd342-e10f-46fd-830f-5b961ebc308d","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","name":"stop-cn5qj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-cn5qj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"stop-cn5qj"}}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf"}
{"level":"error","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d553786c-246c-4921-b3ff-7d3051955dbf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-cn5qj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a"}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a"}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.149Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:26.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.150Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:26.150Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"b51286ac-04f8-4ac4-ba65-99f72202fd4a","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:26.155Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d9aa1249-a301-4670-b2d2-65c2df9b3a17","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:26.289Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ff6ba45c-f73b-4ca4-b962-c255a0bfb0ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:26.289Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ff6ba45c-f73b-4ca4-b962-c255a0bfb0ff","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.289Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ff6ba45c-f73b-4ca4-b962-c255a0bfb0ff","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:26.290Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ff6ba45c-f73b-4ca4-b962-c255a0bfb0ff"}
{"level":"info","ts":"2026-06-11T10:22:26.290Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ff6ba45c-f73b-4ca4-b962-c255a0bfb0ff","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc"}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc"}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.426Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:26.427Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.427Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.427Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.427Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:26.427Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"56c55175-f7e9-4f86-a839-410a7e60a6fc","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:26.460Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"d01693f8-67ac-408c-a59c-f763510556d8","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"newHead":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"}}
{"level":"info","ts":"2026-06-11T10:22:26.893Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"615635b6-f8c2-49ce-b15a-7fd6bffbdb9d","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"f8292b39-0f18-4d0e-8f43-ec6a9aaef01f","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b665afc4-9621-4b56-a4eb-6258f770a9e6","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c475ab5b-db87-4b4d-a973-cfde426ad128","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c475ab5b-db87-4b4d-a973-cfde426ad128","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c475ab5b-db87-4b4d-a973-cfde426ad128","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b665afc4-9621-4b56-a4eb-6258f770a9e6","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c475ab5b-db87-4b4d-a973-cfde426ad128"}
{"level":"info","ts":"2026-06-11T10:22:26.911Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c475ab5b-db87-4b4d-a973-cfde426ad128","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:26.931Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b665afc4-9621-4b56-a4eb-6258f770a9e6","certificate":"ray-head-cert-stop-cn5qj","dnsNames":["stop-cn5qj-head-svc","localhost","stop-cn5qj-head-svc.test-ns-c6srn.svc","stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.28","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:22:26.931Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"f8292b39-0f18-4d0e-8f43-ec6a9aaef01f","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:22:26.946Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"f8292b39-0f18-4d0e-8f43-ec6a9aaef01f","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:22:26.946Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"f8292b39-0f18-4d0e-8f43-ec6a9aaef01f","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:26.955Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b665afc4-9621-4b56-a4eb-6258f770a9e6","certificate":"ray-worker-cert-stop-cn5qj","dnsNames":["stop-cn5qj-worker-svc","localhost","stop-cn5qj-small-group","stop-cn5qj-worker-svc.test-ns-c6srn.svc","stop-cn5qj-worker-svc.test-ns-c6srn.svc.cluster.local","*.stop-cn5qj-worker-svc.test-ns-c6srn.svc","*.stop-cn5qj-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.28","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:22:26.955Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b665afc4-9621-4b56-a4eb-6258f770a9e6"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:27.403Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:27.404Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"c41db8a8-f3fd-403c-acd6-3ce5f466372b","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:28.911Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c"}
{"level":"info","ts":"2026-06-11T10:22:28.911Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c"}
{"level":"info","ts":"2026-06-11T10:22:28.911Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:22:28.912Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2c226e4b-aa3a-411f-9903-25ef72b14a2c","seconds":300}
{"level":"info","ts":"2026-06-11T10:22:29.290Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2569d49c-06ef-4cda-aa7b-837f6568050f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:29.290Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2569d49c-06ef-4cda-aa7b-837f6568050f","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:29.290Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2569d49c-06ef-4cda-aa7b-837f6568050f","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:29.290Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2569d49c-06ef-4cda-aa7b-837f6568050f"}
{"level":"info","ts":"2026-06-11T10:22:29.290Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2569d49c-06ef-4cda-aa7b-837f6568050f","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:32.291Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9f922565-9049-45d9-aebf-b8232c21684b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:32.291Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9f922565-9049-45d9-aebf-b8232c21684b","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:32.291Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9f922565-9049-45d9-aebf-b8232c21684b","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:32.291Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9f922565-9049-45d9-aebf-b8232c21684b"}
{"level":"info","ts":"2026-06-11T10:22:32.291Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9f922565-9049-45d9-aebf-b8232c21684b","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:35.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3e18206f-6e77-487e-b2bf-3b93757a7572","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:35.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3e18206f-6e77-487e-b2bf-3b93757a7572","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:35.292Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3e18206f-6e77-487e-b2bf-3b93757a7572","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:35.292Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3e18206f-6e77-487e-b2bf-3b93757a7572"}
{"level":"info","ts":"2026-06-11T10:22:35.292Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3e18206f-6e77-487e-b2bf-3b93757a7572","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:38.293Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"40b6dd01-99ab-4116-93f9-68592b4e2510","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:38.293Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"40b6dd01-99ab-4116-93f9-68592b4e2510","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:38.293Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"40b6dd01-99ab-4116-93f9-68592b4e2510","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:38.293Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"40b6dd01-99ab-4116-93f9-68592b4e2510"}
{"level":"info","ts":"2026-06-11T10:22:38.293Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"40b6dd01-99ab-4116-93f9-68592b4e2510","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:41.293Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"89ce9b1b-f627-4246-ab62-0c958cf51255","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:41.293Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"89ce9b1b-f627-4246-ab62-0c958cf51255","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:41.293Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"89ce9b1b-f627-4246-ab62-0c958cf51255","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:41.294Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"89ce9b1b-f627-4246-ab62-0c958cf51255"}
{"level":"info","ts":"2026-06-11T10:22:41.294Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"89ce9b1b-f627-4246-ab62-0c958cf51255","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:44.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e3c2d7e0-496a-4319-aa7e-ca0d40f5f2c3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:44.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e3c2d7e0-496a-4319-aa7e-ca0d40f5f2c3","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:44.295Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e3c2d7e0-496a-4319-aa7e-ca0d40f5f2c3","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:44.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e3c2d7e0-496a-4319-aa7e-ca0d40f5f2c3"}
{"level":"info","ts":"2026-06-11T10:22:44.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e3c2d7e0-496a-4319-aa7e-ca0d40f5f2c3","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:47.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"100f9eb7-a477-48eb-8f58-f91c2be3157f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:47.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"100f9eb7-a477-48eb-8f58-f91c2be3157f","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:47.296Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"100f9eb7-a477-48eb-8f58-f91c2be3157f","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:47.296Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"100f9eb7-a477-48eb-8f58-f91c2be3157f"}
{"level":"info","ts":"2026-06-11T10:22:47.296Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"100f9eb7-a477-48eb-8f58-f91c2be3157f","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:50.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"aedd7c5d-7fdf-4630-b8b6-443d2174037e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:50.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"aedd7c5d-7fdf-4630-b8b6-443d2174037e","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:50.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"aedd7c5d-7fdf-4630-b8b6-443d2174037e","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:50.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"aedd7c5d-7fdf-4630-b8b6-443d2174037e"}
{"level":"info","ts":"2026-06-11T10:22:50.297Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"aedd7c5d-7fdf-4630-b8b6-443d2174037e","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:53.298Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"396f3f25-c0da-4cca-aeb5-b1e3568d363d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:53.298Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"396f3f25-c0da-4cca-aeb5-b1e3568d363d","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:53.298Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"396f3f25-c0da-4cca-aeb5-b1e3568d363d","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:53.298Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"396f3f25-c0da-4cca-aeb5-b1e3568d363d"}
{"level":"info","ts":"2026-06-11T10:22:53.298Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"396f3f25-c0da-4cca-aeb5-b1e3568d363d","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:53.529Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"62ff348b-d8c5-4e2c-8188-d14fd3256030","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:53.529Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"62ff348b-d8c5-4e2c-8188-d14fd3256030","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:22:53.529Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"62ff348b-d8c5-4e2c-8188-d14fd3256030","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:22:53.544Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"62ff348b-d8c5-4e2c-8188-d14fd3256030","certificate":"ray-head-cert-stop-cn5qj","dnsNames":["stop-cn5qj-head-svc","localhost","stop-cn5qj-head-svc.test-ns-c6srn.svc","stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.28","10.134.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:22:53.560Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"62ff348b-d8c5-4e2c-8188-d14fd3256030","certificate":"ray-worker-cert-stop-cn5qj","dnsNames":["stop-cn5qj-worker-svc","localhost","stop-cn5qj-small-group","stop-cn5qj-worker-svc.test-ns-c6srn.svc","stop-cn5qj-worker-svc.test-ns-c6srn.svc.cluster.local","*.stop-cn5qj-worker-svc.test-ns-c6srn.svc","*.stop-cn5qj-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.28","10.134.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:22:53.560Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"62ff348b-d8c5-4e2c-8188-d14fd3256030","rayCluster":"stop-cn5qj","podIPs":["10.133.0.28","10.134.0.41"]}
{"level":"info","ts":"2026-06-11T10:22:56.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5a541344-7823-4c0a-bacc-195760414e09","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:56.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5a541344-7823-4c0a-bacc-195760414e09","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:56.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5a541344-7823-4c0a-bacc-195760414e09","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:56.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5a541344-7823-4c0a-bacc-195760414e09"}
{"level":"info","ts":"2026-06-11T10:22:56.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5a541344-7823-4c0a-bacc-195760414e09","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:22:59.301Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d216ad88-f055-4a5f-acf7-45173f738679","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:22:59.301Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d216ad88-f055-4a5f-acf7-45173f738679","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:22:59.301Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d216ad88-f055-4a5f-acf7-45173f738679","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:22:59.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d216ad88-f055-4a5f-acf7-45173f738679"}
{"level":"info","ts":"2026-06-11T10:22:59.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d216ad88-f055-4a5f-acf7-45173f738679","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:02.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3ee479f2-9be9-4a47-affe-64a8ba0b057c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:02.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3ee479f2-9be9-4a47-affe-64a8ba0b057c","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:02.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3ee479f2-9be9-4a47-affe-64a8ba0b057c","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:02.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3ee479f2-9be9-4a47-affe-64a8ba0b057c"}
{"level":"info","ts":"2026-06-11T10:23:02.302Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3ee479f2-9be9-4a47-affe-64a8ba0b057c","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:05.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"98146583-5adc-40eb-8758-c7c3c02ea7f4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:05.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"98146583-5adc-40eb-8758-c7c3c02ea7f4","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:05.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"98146583-5adc-40eb-8758-c7c3c02ea7f4","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:05.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"98146583-5adc-40eb-8758-c7c3c02ea7f4"}
{"level":"info","ts":"2026-06-11T10:23:05.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"98146583-5adc-40eb-8758-c7c3c02ea7f4","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:08.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22ffe614-fcfc-40be-ab0f-353ae163e441","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:08.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22ffe614-fcfc-40be-ab0f-353ae163e441","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:08.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22ffe614-fcfc-40be-ab0f-353ae163e441","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:08.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22ffe614-fcfc-40be-ab0f-353ae163e441"}
{"level":"info","ts":"2026-06-11T10:23:08.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22ffe614-fcfc-40be-ab0f-353ae163e441","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:11.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"a99f5c66-6582-4e7d-9968-d06e33d76440","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:11.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"a99f5c66-6582-4e7d-9968-d06e33d76440","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:11.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"a99f5c66-6582-4e7d-9968-d06e33d76440","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:11.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"a99f5c66-6582-4e7d-9968-d06e33d76440"}
{"level":"info","ts":"2026-06-11T10:23:11.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"a99f5c66-6582-4e7d-9968-d06e33d76440","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:14.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0e28cb55-caaa-430e-9adf-48c3f31e4836","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:14.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0e28cb55-caaa-430e-9adf-48c3f31e4836","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:14.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0e28cb55-caaa-430e-9adf-48c3f31e4836","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:14.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0e28cb55-caaa-430e-9adf-48c3f31e4836"}
{"level":"info","ts":"2026-06-11T10:23:14.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0e28cb55-caaa-430e-9adf-48c3f31e4836","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:16.588Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"622652b0-3ba0-495c-91b0-a3134b42845e","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:23:16.589Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"622652b0-3ba0-495c-91b0-a3134b42845e","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:23:16.589Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"622652b0-3ba0-495c-91b0-a3134b42845e","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:23:16.589Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"622652b0-3ba0-495c-91b0-a3134b42845e","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:23:17.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2c59b72f-6bfd-4985-803f-05cf0e69fa37","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:17.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2c59b72f-6bfd-4985-803f-05cf0e69fa37","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:17.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2c59b72f-6bfd-4985-803f-05cf0e69fa37","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:17.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2c59b72f-6bfd-4985-803f-05cf0e69fa37"}
{"level":"info","ts":"2026-06-11T10:23:17.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2c59b72f-6bfd-4985-803f-05cf0e69fa37","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:20.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2ff3009c-8ddb-4454-b0f3-fed60e1606c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:20.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2ff3009c-8ddb-4454-b0f3-fed60e1606c0","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:20.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2ff3009c-8ddb-4454-b0f3-fed60e1606c0","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:20.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2ff3009c-8ddb-4454-b0f3-fed60e1606c0"}
{"level":"info","ts":"2026-06-11T10:23:20.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"2ff3009c-8ddb-4454-b0f3-fed60e1606c0","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:23.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1ca82aa9-f152-4bd2-92df-908ec390c8d0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:23.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1ca82aa9-f152-4bd2-92df-908ec390c8d0","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:23.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1ca82aa9-f152-4bd2-92df-908ec390c8d0","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:23.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1ca82aa9-f152-4bd2-92df-908ec390c8d0"}
{"level":"info","ts":"2026-06-11T10:23:23.309Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1ca82aa9-f152-4bd2-92df-908ec390c8d0","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:26.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"fcaaa7d3-9476-4315-abeb-4d84d20e9181","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:26.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"fcaaa7d3-9476-4315-abeb-4d84d20e9181","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:26.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"fcaaa7d3-9476-4315-abeb-4d84d20e9181","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:26.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"fcaaa7d3-9476-4315-abeb-4d84d20e9181"}
{"level":"info","ts":"2026-06-11T10:23:26.309Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"fcaaa7d3-9476-4315-abeb-4d84d20e9181","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:29.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4095da7b-1fc0-4f9c-a5e0-f45a162b044f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:29.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4095da7b-1fc0-4f9c-a5e0-f45a162b044f","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:29.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4095da7b-1fc0-4f9c-a5e0-f45a162b044f","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:29.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4095da7b-1fc0-4f9c-a5e0-f45a162b044f"}
{"level":"info","ts":"2026-06-11T10:23:29.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"4095da7b-1fc0-4f9c-a5e0-f45a162b044f","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:32.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"303059f8-65d3-46df-b3ca-fdbc627bb937","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:32.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"303059f8-65d3-46df-b3ca-fdbc627bb937","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:32.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"303059f8-65d3-46df-b3ca-fdbc627bb937","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:32.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"303059f8-65d3-46df-b3ca-fdbc627bb937"}
{"level":"info","ts":"2026-06-11T10:23:32.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"303059f8-65d3-46df-b3ca-fdbc627bb937","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:35.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"96a2e7c6-e10a-41eb-8d32-4e78cb495839","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:35.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"96a2e7c6-e10a-41eb-8d32-4e78cb495839","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:35.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"96a2e7c6-e10a-41eb-8d32-4e78cb495839","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:35.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"96a2e7c6-e10a-41eb-8d32-4e78cb495839"}
{"level":"info","ts":"2026-06-11T10:23:35.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"96a2e7c6-e10a-41eb-8d32-4e78cb495839","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:38.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df218e56-6ae4-4d22-bf20-08bba20dc387","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:38.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df218e56-6ae4-4d22-bf20-08bba20dc387","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:38.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df218e56-6ae4-4d22-bf20-08bba20dc387","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:38.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df218e56-6ae4-4d22-bf20-08bba20dc387"}
{"level":"info","ts":"2026-06-11T10:23:38.312Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df218e56-6ae4-4d22-bf20-08bba20dc387","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:41.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f2a64806-f2dd-47a9-9aec-0058a360b451","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:41.313Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f2a64806-f2dd-47a9-9aec-0058a360b451","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:41.313Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f2a64806-f2dd-47a9-9aec-0058a360b451","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:41.313Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f2a64806-f2dd-47a9-9aec-0058a360b451"}
{"level":"info","ts":"2026-06-11T10:23:41.313Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f2a64806-f2dd-47a9-9aec-0058a360b451","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:44.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"efcea36f-4d96-4bf4-8d8b-ef94a2066884","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:44.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"efcea36f-4d96-4bf4-8d8b-ef94a2066884","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:44.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"efcea36f-4d96-4bf4-8d8b-ef94a2066884","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:44.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"efcea36f-4d96-4bf4-8d8b-ef94a2066884"}
{"level":"info","ts":"2026-06-11T10:23:44.314Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"efcea36f-4d96-4bf4-8d8b-ef94a2066884","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:47.315Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bfb00ffd-736c-47f1-974b-8589c04d5738","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:47.315Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bfb00ffd-736c-47f1-974b-8589c04d5738","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:47.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bfb00ffd-736c-47f1-974b-8589c04d5738","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:47.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bfb00ffd-736c-47f1-974b-8589c04d5738"}
{"level":"info","ts":"2026-06-11T10:23:47.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bfb00ffd-736c-47f1-974b-8589c04d5738","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:50.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"76acc2ad-7ed1-4038-9bf2-5b0003da8c5c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:50.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"76acc2ad-7ed1-4038-9bf2-5b0003da8c5c","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:50.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"76acc2ad-7ed1-4038-9bf2-5b0003da8c5c","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:50.316Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"76acc2ad-7ed1-4038-9bf2-5b0003da8c5c"}
{"level":"info","ts":"2026-06-11T10:23:50.316Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"76acc2ad-7ed1-4038-9bf2-5b0003da8c5c","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:53.317Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"24d02417-6984-4ac2-8272-c62c911c1498","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:53.317Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"24d02417-6984-4ac2-8272-c62c911c1498","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:53.317Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"24d02417-6984-4ac2-8272-c62c911c1498","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:53.317Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"24d02417-6984-4ac2-8272-c62c911c1498"}
{"level":"info","ts":"2026-06-11T10:23:53.317Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"24d02417-6984-4ac2-8272-c62c911c1498","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:53.561Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0c601f3a-2cda-4bda-96ca-6d79c4f65348","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:53.561Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0c601f3a-2cda-4bda-96ca-6d79c4f65348","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:23:53.561Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0c601f3a-2cda-4bda-96ca-6d79c4f65348","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:23:53.561Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0c601f3a-2cda-4bda-96ca-6d79c4f65348","rayCluster":"stop-cn5qj","podIPs":["10.133.0.28","10.134.0.41"]}
{"level":"info","ts":"2026-06-11T10:23:56.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bae43d87-be90-471e-a5bc-fe6891746088","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:56.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bae43d87-be90-471e-a5bc-fe6891746088","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:56.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bae43d87-be90-471e-a5bc-fe6891746088","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:56.318Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bae43d87-be90-471e-a5bc-fe6891746088"}
{"level":"info","ts":"2026-06-11T10:23:56.318Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bae43d87-be90-471e-a5bc-fe6891746088","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:23:59.319Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7378ffe7-4ff6-46eb-be2c-ed448d07db41","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:23:59.319Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7378ffe7-4ff6-46eb-be2c-ed448d07db41","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:23:59.319Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7378ffe7-4ff6-46eb-be2c-ed448d07db41","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:23:59.320Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7378ffe7-4ff6-46eb-be2c-ed448d07db41"}
{"level":"info","ts":"2026-06-11T10:23:59.320Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7378ffe7-4ff6-46eb-be2c-ed448d07db41","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:02.243Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85"}
{"level":"info","ts":"2026-06-11T10:24:02.243Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85"}
{"level":"info","ts":"2026-06-11T10:24:02.243Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","podName":"stop-cn5qj-head-9ljzg","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:24:02.244Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"0a79d1d9-802a-4812-98b7-368ec20c5b85","seconds":300}
{"level":"info","ts":"2026-06-11T10:24:02.320Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c9ede9a6-2a73-48f6-817a-dc475b3da0f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:02.320Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c9ede9a6-2a73-48f6-817a-dc475b3da0f1","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:02.320Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c9ede9a6-2a73-48f6-817a-dc475b3da0f1","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:02.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c9ede9a6-2a73-48f6-817a-dc475b3da0f1"}
{"level":"info","ts":"2026-06-11T10:24:02.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"c9ede9a6-2a73-48f6-817a-dc475b3da0f1","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:03.240Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc"}
{"level":"info","ts":"2026-06-11T10:24:03.240Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc"}
{"level":"info","ts":"2026-06-11T10:24:03.240Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:03.240Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","podName":"stop-cn5qj-head-9ljzg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","podName":"stop-cn5qj-head-9ljzg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:24:03.241Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"00d2969c-09b8-49a4-8c15-cfd9a697b5bc","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:22:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ca872dc3-e53f-4c71-9555-8c1b3bf92302","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1dad8af0-9bb2-4394-92c6-536eac16c03c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1dad8af0-9bb2-4394-92c6-536eac16c03c","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"57e046ba-239e-4038-91c8-e62a3f36149e","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1dad8af0-9bb2-4394-92c6-536eac16c03c","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ca872dc3-e53f-4c71-9555-8c1b3bf92302","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ca872dc3-e53f-4c71-9555-8c1b3bf92302","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ca872dc3-e53f-4c71-9555-8c1b3bf92302","rayCluster":"stop-cn5qj","podIPs":["10.133.0.28","10.134.0.41"]}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1dad8af0-9bb2-4394-92c6-536eac16c03c"}
{"level":"info","ts":"2026-06-11T10:24:03.263Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1dad8af0-9bb2-4394-92c6-536eac16c03c","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:03.289Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"57e046ba-239e-4038-91c8-e62a3f36149e","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:24:03.303Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"57e046ba-239e-4038-91c8-e62a3f36149e","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:24:03.303Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"57e046ba-239e-4038-91c8-e62a3f36149e","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773"}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773"}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","podName":"stop-cn5qj-head-9ljzg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","podName":"stop-cn5qj-head-9ljzg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:24:05.263Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"5283371e-89cf-429a-813b-5937fbbe8773","seconds":300}
{"level":"info","ts":"2026-06-11T10:24:05.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1e03fce1-72bd-4250-a0b6-78301db29bda","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:05.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1e03fce1-72bd-4250-a0b6-78301db29bda","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:05.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1e03fce1-72bd-4250-a0b6-78301db29bda","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:05.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1e03fce1-72bd-4250-a0b6-78301db29bda"}
{"level":"info","ts":"2026-06-11T10:24:05.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"1e03fce1-72bd-4250-a0b6-78301db29bda","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:08.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e9d90b8e-dc5f-4523-aa99-97e02771c294","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:08.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e9d90b8e-dc5f-4523-aa99-97e02771c294","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:08.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e9d90b8e-dc5f-4523-aa99-97e02771c294","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:08.322Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e9d90b8e-dc5f-4523-aa99-97e02771c294"}
{"level":"info","ts":"2026-06-11T10:24:08.322Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"e9d90b8e-dc5f-4523-aa99-97e02771c294","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:11.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"abfc03c7-799e-4338-a119-227102de9e9e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:11.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"abfc03c7-799e-4338-a119-227102de9e9e","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:11.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"abfc03c7-799e-4338-a119-227102de9e9e","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:11.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"abfc03c7-799e-4338-a119-227102de9e9e"}
{"level":"info","ts":"2026-06-11T10:24:11.323Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"abfc03c7-799e-4338-a119-227102de9e9e","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c"}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c"}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:13.885Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:13.886Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","podName":"stop-cn5qj-head-9ljzg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:24:13.886Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","podName":"stop-cn5qj-head-9ljzg","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:24:13.886Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:24:13.886Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"bfeaf856-a9b5-41c7-949c-82f1e248a61c","seconds":300}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:24:14.278Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"88c48ee2-ed41-4a70-8a35-e4c05f8de810","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ae932c3f-aa0c-4d8d-a1b6-d960e57d09c4","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6a1012a8-2a90-435d-bb4c-89968174ab9b","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f7379df6-0c90-4be9-8a65-e2ea90bc2fad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f7379df6-0c90-4be9-8a65-e2ea90bc2fad","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f7379df6-0c90-4be9-8a65-e2ea90bc2fad","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6a1012a8-2a90-435d-bb4c-89968174ab9b","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6a1012a8-2a90-435d-bb4c-89968174ab9b","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6a1012a8-2a90-435d-bb4c-89968174ab9b","rayCluster":"stop-cn5qj","podIPs":["10.134.0.41","10.133.0.28"]}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f7379df6-0c90-4be9-8a65-e2ea90bc2fad"}
{"level":"info","ts":"2026-06-11T10:24:14.293Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f7379df6-0c90-4be9-8a65-e2ea90bc2fad","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:14.311Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ae932c3f-aa0c-4d8d-a1b6-d960e57d09c4","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:24:14.323Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ae932c3f-aa0c-4d8d-a1b6-d960e57d09c4","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:24:14.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7dfe2a6e-0fc9-4e3b-a10e-c1e461b81e6b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:14.323Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ae932c3f-aa0c-4d8d-a1b6-d960e57d09c4","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7dfe2a6e-0fc9-4e3b-a10e-c1e461b81e6b","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7dfe2a6e-0fc9-4e3b-a10e-c1e461b81e6b","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:14.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7dfe2a6e-0fc9-4e3b-a10e-c1e461b81e6b"}
{"level":"info","ts":"2026-06-11T10:24:14.323Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7dfe2a6e-0fc9-4e3b-a10e-c1e461b81e6b","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa"}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa"}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:24:14.892Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"f2b059b3-20b9-457c-b4d0-6e23304df2aa","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6808ce2b-900d-474d-9543-e4c58ce26875","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"475e1fd9-7221-4695-b97e-72d462d9b045","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"513c0902-3430-4c58-b58d-e266e78e2f05","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"513c0902-3430-4c58-b58d-e266e78e2f05","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"513c0902-3430-4c58-b58d-e266e78e2f05","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"475e1fd9-7221-4695-b97e-72d462d9b045","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"475e1fd9-7221-4695-b97e-72d462d9b045","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"475e1fd9-7221-4695-b97e-72d462d9b045","rayCluster":"stop-cn5qj","podIPs":["10.133.0.28","10.134.0.41"]}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"513c0902-3430-4c58-b58d-e266e78e2f05"}
{"level":"info","ts":"2026-06-11T10:24:14.911Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"513c0902-3430-4c58-b58d-e266e78e2f05","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:14.927Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6808ce2b-900d-474d-9543-e4c58ce26875","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:24:14.943Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6808ce2b-900d-474d-9543-e4c58ce26875","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:24:14.943Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"6808ce2b-900d-474d-9543-e4c58ce26875","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086"}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086"}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","podName":"stop-cn5qj-small-group-worker-xvwbb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:22:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:24:16.293Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"a3166aa2-5090-466f-9ca6-078c28221086","seconds":300}
{"level":"info","ts":"2026-06-11T10:24:16.589Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f746199f-6444-44e6-a0d0-0c48f9b825db","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:24:16.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f746199f-6444-44e6-a0d0-0c48f9b825db","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:24:16.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f746199f-6444-44e6-a0d0-0c48f9b825db","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:24:16.590Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"f746199f-6444-44e6-a0d0-0c48f9b825db","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:24:17.325Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5ffc1cd5-dd2f-4b48-965b-d9b428cbaec6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:17.325Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5ffc1cd5-dd2f-4b48-965b-d9b428cbaec6","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:17.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5ffc1cd5-dd2f-4b48-965b-d9b428cbaec6","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:17.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5ffc1cd5-dd2f-4b48-965b-d9b428cbaec6"}
{"level":"info","ts":"2026-06-11T10:24:17.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5ffc1cd5-dd2f-4b48-965b-d9b428cbaec6","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:20.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22738e4e-7680-4f5c-b30d-5fbfef440e25","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:20.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22738e4e-7680-4f5c-b30d-5fbfef440e25","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:20.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22738e4e-7680-4f5c-b30d-5fbfef440e25","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:20.326Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22738e4e-7680-4f5c-b30d-5fbfef440e25"}
{"level":"info","ts":"2026-06-11T10:24:20.326Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"22738e4e-7680-4f5c-b30d-5fbfef440e25","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:23.327Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"59052247-11f3-4f6b-92cd-1f2dfb895f80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:23.327Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"59052247-11f3-4f6b-92cd-1f2dfb895f80","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:23.327Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"59052247-11f3-4f6b-92cd-1f2dfb895f80","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:23.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"59052247-11f3-4f6b-92cd-1f2dfb895f80"}
{"level":"info","ts":"2026-06-11T10:24:23.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"59052247-11f3-4f6b-92cd-1f2dfb895f80","RayCluster":"stop-cn5qj","State":""}
{"level":"info","ts":"2026-06-11T10:24:25.909Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21"}
{"level":"info","ts":"2026-06-11T10:24:25.909Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21"}
{"level":"info","ts":"2026-06-11T10:24:25.909Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:25.909Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-11T10:24:25.910Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"2d40ce15-c6d0-4af7-8dc4-80e1d96e0d21","name":"stop-cn5qj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:24:25.923Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"5c18b601-5299-4132-b61a-beb0fa977be5","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"d27ef09f-a767-40de-bcf3-c7d59bc4a8b6","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"d27ef09f-a767-40de-bcf3-c7d59bc4a8b6","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"d27ef09f-a767-40de-bcf3-c7d59bc4a8b6","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"d27ef09f-a767-40de-bcf3-c7d59bc4a8b6","rayCluster":"stop-cn5qj","podIPs":["10.133.0.28","10.134.0.41"]}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","head service name":"stop-cn5qj-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","head service URL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e"}
{"level":"info","ts":"2026-06-11T10:24:25.924Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","command":["if","!","ray","job","status","--address","http://stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","stop-hrfcm",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","--no-wait","--submission-id","stop-hrfcm","--","python /home/ray/jobs/stop.py",";","fi",";","ray","job","logs","--address","http://stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","--follow","stop-hrfcm"]}
{"level":"info","ts":"2026-06-11T10:24:25.933Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-06-11T10:24:25.933Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","SubmissionMode":"K8sJobMode","RayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:24:25.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73568012-5b23-4d08-9f46-2a3dcc75781e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:24:25.937Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"5c18b601-5299-4132-b61a-beb0fa977be5","name":"stop-cn5qj-head"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","head service name":"stop-cn5qj-head-svc","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","head service URL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","SubmissionMode":"K8sJobMode","RayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:24:25.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:24:25.952Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"5c18b601-5299-4132-b61a-beb0fa977be5","name":"stop-cn5qj-workers"}
{"level":"info","ts":"2026-06-11T10:24:25.952Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"5c18b601-5299-4132-b61a-beb0fa977be5","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.958Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:24:25.958Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2"}
{"level":"error","ts":"2026-06-11T10:24:25.958Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"65754711-e269-41c7-88fe-20d270ca5ce2","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:25.958Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"125fc3a9-3fa4-4fde-aed8-6ce0bf54e213","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:25.958Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"125fc3a9-3fa4-4fde-aed8-6ce0bf54e213","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.958Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"125fc3a9-3fa4-4fde-aed8-6ce0bf54e213","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:25.958Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"125fc3a9-3fa4-4fde-aed8-6ce0bf54e213"}
{"level":"error","ts":"2026-06-11T10:24:25.967Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"125fc3a9-3fa4-4fde-aed8-6ce0bf54e213","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:25.967Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"125fc3a9-3fa4-4fde-aed8-6ce0bf54e213"}
{"level":"error","ts":"2026-06-11T10:24:25.967Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"125fc3a9-3fa4-4fde-aed8-6ce0bf54e213","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:25.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ffaa0ac4-85f8-4c28-ba25-b9ab91bf05ce","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:25.968Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ffaa0ac4-85f8-4c28-ba25-b9ab91bf05ce","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.968Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ffaa0ac4-85f8-4c28-ba25-b9ab91bf05ce","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:25.968Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ffaa0ac4-85f8-4c28-ba25-b9ab91bf05ce"}
{"level":"error","ts":"2026-06-11T10:24:25.972Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ffaa0ac4-85f8-4c28-ba25-b9ab91bf05ce","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:25.972Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ffaa0ac4-85f8-4c28-ba25-b9ab91bf05ce"}
{"level":"error","ts":"2026-06-11T10:24:25.972Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ffaa0ac4-85f8-4c28-ba25-b9ab91bf05ce","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:25.972Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"31049279-1c2a-4388-adc7-b61b321f8c2f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:25.972Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"31049279-1c2a-4388-adc7-b61b321f8c2f","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.972Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"31049279-1c2a-4388-adc7-b61b321f8c2f","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:25.973Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"31049279-1c2a-4388-adc7-b61b321f8c2f"}
{"level":"error","ts":"2026-06-11T10:24:25.977Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"31049279-1c2a-4388-adc7-b61b321f8c2f","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:25.977Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"31049279-1c2a-4388-adc7-b61b321f8c2f"}
{"level":"error","ts":"2026-06-11T10:24:25.977Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"31049279-1c2a-4388-adc7-b61b321f8c2f","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:25.978Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"06db5ee9-0d1e-4020-a360-5ec57afe9ccf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:25.978Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"06db5ee9-0d1e-4020-a360-5ec57afe9ccf","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:25.978Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"06db5ee9-0d1e-4020-a360-5ec57afe9ccf","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:25.979Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"06db5ee9-0d1e-4020-a360-5ec57afe9ccf"}
{"level":"error","ts":"2026-06-11T10:24:26.031Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"06db5ee9-0d1e-4020-a360-5ec57afe9ccf","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.031Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"06db5ee9-0d1e-4020-a360-5ec57afe9ccf"}
{"level":"error","ts":"2026-06-11T10:24:26.031Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"06db5ee9-0d1e-4020-a360-5ec57afe9ccf","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.112Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0a314d60-1a60-4a3a-a22c-40444bcfa04f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:26.112Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0a314d60-1a60-4a3a-a22c-40444bcfa04f","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:26.112Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0a314d60-1a60-4a3a-a22c-40444bcfa04f","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:26.112Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0a314d60-1a60-4a3a-a22c-40444bcfa04f"}
{"level":"error","ts":"2026-06-11T10:24:26.117Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0a314d60-1a60-4a3a-a22c-40444bcfa04f","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.117Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0a314d60-1a60-4a3a-a22c-40444bcfa04f"}
{"level":"error","ts":"2026-06-11T10:24:26.117Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0a314d60-1a60-4a3a-a22c-40444bcfa04f","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.278Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73e9f8d6-853e-4782-8689-080cbf3da028","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:26.278Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73e9f8d6-853e-4782-8689-080cbf3da028","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:26.278Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73e9f8d6-853e-4782-8689-080cbf3da028","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:26.278Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73e9f8d6-853e-4782-8689-080cbf3da028"}
{"level":"error","ts":"2026-06-11T10:24:26.283Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73e9f8d6-853e-4782-8689-080cbf3da028","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.283Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73e9f8d6-853e-4782-8689-080cbf3da028"}
{"level":"error","ts":"2026-06-11T10:24:26.283Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"73e9f8d6-853e-4782-8689-080cbf3da028","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.604Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d0c43bc6-8279-43d8-9016-d431c0c9ab9d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:26.604Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d0c43bc6-8279-43d8-9016-d431c0c9ab9d","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:26.604Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d0c43bc6-8279-43d8-9016-d431c0c9ab9d","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:26.604Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d0c43bc6-8279-43d8-9016-d431c0c9ab9d"}
{"level":"error","ts":"2026-06-11T10:24:26.609Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d0c43bc6-8279-43d8-9016-d431c0c9ab9d","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.609Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d0c43bc6-8279-43d8-9016-d431c0c9ab9d"}
{"level":"error","ts":"2026-06-11T10:24:26.609Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"d0c43bc6-8279-43d8-9016-d431c0c9ab9d","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0d2651e1-0351-4e7c-91c2-13e09faab4ae","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:26.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0d2651e1-0351-4e7c-91c2-13e09faab4ae","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:26.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0d2651e1-0351-4e7c-91c2-13e09faab4ae","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:26.971Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0d2651e1-0351-4e7c-91c2-13e09faab4ae"}
{"level":"error","ts":"2026-06-11T10:24:26.976Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0d2651e1-0351-4e7c-91c2-13e09faab4ae","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:26.976Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0d2651e1-0351-4e7c-91c2-13e09faab4ae"}
{"level":"error","ts":"2026-06-11T10:24:26.976Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0d2651e1-0351-4e7c-91c2-13e09faab4ae","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:27.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"53941128-d5eb-4ea1-a66a-25032e564bcb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:27.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"53941128-d5eb-4ea1-a66a-25032e564bcb","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:27.251Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"53941128-d5eb-4ea1-a66a-25032e564bcb","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:27.251Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"53941128-d5eb-4ea1-a66a-25032e564bcb"}
{"level":"error","ts":"2026-06-11T10:24:27.256Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"53941128-d5eb-4ea1-a66a-25032e564bcb","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:27.256Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"53941128-d5eb-4ea1-a66a-25032e564bcb"}
{"level":"error","ts":"2026-06-11T10:24:27.256Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"53941128-d5eb-4ea1-a66a-25032e564bcb","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4"}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4"}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","1 head service found":"stop-cn5qj-head-svc"}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","Found 1 head Pod":"stop-cn5qj-head-9ljzg","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","head Pod":"stop-cn5qj-head-9ljzg","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-cn5qj-head-9ljzg. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","worker Pod":"stop-cn5qj-small-group-worker-xvwbb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-cn5qj-small-group-worker-xvwbb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:24:27.924Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:24:27.925Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"9543c141-1055-4495-b5e7-9c1aed74e5b4","seconds":300}
{"level":"info","ts":"2026-06-11T10:24:29.817Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"599e180f-ad1b-4e74-b111-3fc9d7a289f2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:29.817Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"599e180f-ad1b-4e74-b111-3fc9d7a289f2","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:29.817Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"599e180f-ad1b-4e74-b111-3fc9d7a289f2","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:29.817Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"599e180f-ad1b-4e74-b111-3fc9d7a289f2"}
{"level":"error","ts":"2026-06-11T10:24:29.822Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"599e180f-ad1b-4e74-b111-3fc9d7a289f2","JobId":"stop-hrfcm","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:29.822Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"599e180f-ad1b-4e74-b111-3fc9d7a289f2"}
{"level":"error","ts":"2026-06-11T10:24:29.822Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"599e180f-ad1b-4e74-b111-3fc9d7a289f2","error":"Job stop-hrfcm does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:24:34.943Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"34a335b6-0c1f-42ce-8b4f-b400d37dc5c8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:34.943Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"34a335b6-0c1f-42ce-8b4f-b400d37dc5c8","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:34.943Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"34a335b6-0c1f-42ce-8b4f-b400d37dc5c8","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:34.943Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"34a335b6-0c1f-42ce-8b4f-b400d37dc5c8"}
{"level":"info","ts":"2026-06-11T10:24:34.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"34a335b6-0c1f-42ce-8b4f-b400d37dc5c8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:34.950Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"34a335b6-0c1f-42ce-8b4f-b400d37dc5c8","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:24:34.964Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"be1f278f-8555-4569-a8c3-5c45b5fdf2a1","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:34.964Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"be1f278f-8555-4569-a8c3-5c45b5fdf2a1","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:34.964Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"be1f278f-8555-4569-a8c3-5c45b5fdf2a1","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:34.964Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"be1f278f-8555-4569-a8c3-5c45b5fdf2a1"}
{"level":"info","ts":"2026-06-11T10:24:34.968Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"be1f278f-8555-4569-a8c3-5c45b5fdf2a1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:37.964Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5863b9a9-eb20-4542-a430-dcb5567ea023","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:37.964Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5863b9a9-eb20-4542-a430-dcb5567ea023","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:37.964Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5863b9a9-eb20-4542-a430-dcb5567ea023","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:37.965Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5863b9a9-eb20-4542-a430-dcb5567ea023"}
{"level":"info","ts":"2026-06-11T10:24:37.968Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5863b9a9-eb20-4542-a430-dcb5567ea023","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:37.968Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"5863b9a9-eb20-4542-a430-dcb5567ea023","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:24:37.985Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9188cb87-a6e2-4958-a094-adf07031cea4","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:37.985Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9188cb87-a6e2-4958-a094-adf07031cea4","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:37.985Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9188cb87-a6e2-4958-a094-adf07031cea4","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:37.985Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9188cb87-a6e2-4958-a094-adf07031cea4"}
{"level":"info","ts":"2026-06-11T10:24:37.988Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"9188cb87-a6e2-4958-a094-adf07031cea4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:40.984Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ca44890c-3d18-41d7-89eb-5b3f3b54e457","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:40.984Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ca44890c-3d18-41d7-89eb-5b3f3b54e457","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:40.984Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ca44890c-3d18-41d7-89eb-5b3f3b54e457","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:40.984Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ca44890c-3d18-41d7-89eb-5b3f3b54e457"}
{"level":"info","ts":"2026-06-11T10:24:40.994Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"ca44890c-3d18-41d7-89eb-5b3f3b54e457","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:43.995Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df5743f3-45e1-4123-b8b7-9d03908422db","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:43.995Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df5743f3-45e1-4123-b8b7-9d03908422db","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:43.995Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df5743f3-45e1-4123-b8b7-9d03908422db","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:43.995Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df5743f3-45e1-4123-b8b7-9d03908422db"}
{"level":"info","ts":"2026-06-11T10:24:43.998Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"df5743f3-45e1-4123-b8b7-9d03908422db","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:46.998Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"8b3029ff-7cd2-40a6-a455-5a563913643a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:46.998Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"8b3029ff-7cd2-40a6-a455-5a563913643a","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:46.998Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"8b3029ff-7cd2-40a6-a455-5a563913643a","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:46.998Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"8b3029ff-7cd2-40a6-a455-5a563913643a"}
{"level":"info","ts":"2026-06-11T10:24:47.002Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"8b3029ff-7cd2-40a6-a455-5a563913643a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:50.003Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f4272567-78be-4eab-a354-030556ba8395","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:50.003Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f4272567-78be-4eab-a354-030556ba8395","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:50.003Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f4272567-78be-4eab-a354-030556ba8395","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:50.003Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f4272567-78be-4eab-a354-030556ba8395"}
{"level":"info","ts":"2026-06-11T10:24:50.006Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f4272567-78be-4eab-a354-030556ba8395","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:53.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"226c2b02-5e7a-42db-b154-bb895e84cf98","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:53.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"226c2b02-5e7a-42db-b154-bb895e84cf98","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:53.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"226c2b02-5e7a-42db-b154-bb895e84cf98","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:53.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"226c2b02-5e7a-42db-b154-bb895e84cf98"}
{"level":"info","ts":"2026-06-11T10:24:53.011Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"226c2b02-5e7a-42db-b154-bb895e84cf98","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:53.562Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0858f51a-9014-4a85-b734-a807e2961d30","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:53.562Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0858f51a-9014-4a85-b734-a807e2961d30","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:24:53.562Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0858f51a-9014-4a85-b734-a807e2961d30","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:24:53.562Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"0858f51a-9014-4a85-b734-a807e2961d30","rayCluster":"stop-cn5qj","podIPs":["10.133.0.28","10.134.0.41"]}
{"level":"info","ts":"2026-06-11T10:24:56.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"833d6bdc-c92f-49ef-b7a8-aa6bbaebe3cb","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:56.012Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"833d6bdc-c92f-49ef-b7a8-aa6bbaebe3cb","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:56.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"833d6bdc-c92f-49ef-b7a8-aa6bbaebe3cb","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:56.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"833d6bdc-c92f-49ef-b7a8-aa6bbaebe3cb"}
{"level":"info","ts":"2026-06-11T10:24:56.015Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"833d6bdc-c92f-49ef-b7a8-aa6bbaebe3cb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:56.015Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"833d6bdc-c92f-49ef-b7a8-aa6bbaebe3cb","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:24:56.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"801a7b3f-d355-4acc-ab9f-013ab1853dba","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:56.036Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"801a7b3f-d355-4acc-ab9f-013ab1853dba","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:56.036Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"801a7b3f-d355-4acc-ab9f-013ab1853dba","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:56.036Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"801a7b3f-d355-4acc-ab9f-013ab1853dba"}
{"level":"info","ts":"2026-06-11T10:24:56.044Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"801a7b3f-d355-4acc-ab9f-013ab1853dba","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:24:59.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3150fbfe-6b71-4d73-b3c2-c8a15b93723e","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:24:59.036Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3150fbfe-6b71-4d73-b3c2-c8a15b93723e","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:24:59.036Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3150fbfe-6b71-4d73-b3c2-c8a15b93723e","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:24:59.037Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3150fbfe-6b71-4d73-b3c2-c8a15b93723e"}
{"level":"info","ts":"2026-06-11T10:24:59.040Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"3150fbfe-6b71-4d73-b3c2-c8a15b93723e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:01.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"03df153b-2596-459a-bcfb-343900f796d1","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:01.099Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"03df153b-2596-459a-bcfb-343900f796d1","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:01.099Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"03df153b-2596-459a-bcfb-343900f796d1","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:01.099Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"03df153b-2596-459a-bcfb-343900f796d1"}
{"level":"info","ts":"2026-06-11T10:25:01.103Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"03df153b-2596-459a-bcfb-343900f796d1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:02.040Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"dafadcd8-a304-4c03-8936-604fb3c1dea7","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:02.040Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"dafadcd8-a304-4c03-8936-604fb3c1dea7","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.040Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"dafadcd8-a304-4c03-8936-604fb3c1dea7","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.041Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"dafadcd8-a304-4c03-8936-604fb3c1dea7"}
{"level":"info","ts":"2026-06-11T10:25:02.044Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"dafadcd8-a304-4c03-8936-604fb3c1dea7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:02.109Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"253a3546-7b8a-43f5-b77a-b46996d62209","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:02.109Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"253a3546-7b8a-43f5-b77a-b46996d62209","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.109Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"253a3546-7b8a-43f5-b77a-b46996d62209","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.109Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"253a3546-7b8a-43f5-b77a-b46996d62209"}
{"level":"info","ts":"2026-06-11T10:25:02.112Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"253a3546-7b8a-43f5-b77a-b46996d62209","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:02.129Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb6b07d7-780b-4a20-acae-cdf3c4fa4cb7","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:02.129Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb6b07d7-780b-4a20-acae-cdf3c4fa4cb7","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.129Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb6b07d7-780b-4a20-acae-cdf3c4fa4cb7","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.129Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb6b07d7-780b-4a20-acae-cdf3c4fa4cb7"}
{"level":"info","ts":"2026-06-11T10:25:02.132Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb6b07d7-780b-4a20-acae-cdf3c4fa4cb7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:24:33Z","endTime":"2026-06-11T10:24:55Z"},"jobId":"stop-hrfcm","rayClusterName":"stop-cn5qj","dashboardURL":"stop-cn5qj-head-svc.test-ns-c6srn.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-06-11T10:22:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:24:25Z","stateTransitionTimes":{"ready":"2026-06-11T10:24:25Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.28","serviceIP":"10.133.0.28","podName":"stop-cn5qj-head-9ljzg","serviceName":"stop-cn5qj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:24:14Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:24:25Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:22:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:22:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:02.132Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"bb6b07d7-780b-4a20-acae-cdf3c4fa4cb7","old JobStatus":"STOPPED","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-11T10:25:02.149Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"a4d35967-8ae5-4bff-9994-b624bc74a292","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:02.149Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"a4d35967-8ae5-4bff-9994-b624bc74a292","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:25:02 +0000 UTC","Now":"2026-06-11T10:25:02.149Z","ShutdownTime":"2026-06-11T10:25:02.000Z"}
{"level":"info","ts":"2026-06-11T10:25:02.363Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"8f17925b-4843-4b2c-8616-d69b2139bca2","DeletionTimestamp":"2026-06-11 10:25:02 +0000 UTC"}
{"level":"info","ts":"2026-06-11T10:25:02.363Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"8f17925b-4843-4b2c-8616-d69b2139bca2","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-06-11T10:25:02.377Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"d837fe14-69f6-4e45-bda1-c50e4069c9a9","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.377Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"d837fe14-69f6-4e45-bda1-c50e4069c9a9"}
{"level":"error","ts":"2026-06-11T10:25:02.377Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"d837fe14-69f6-4e45-bda1-c50e4069c9a9","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.378Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"f95d4457-048f-4250-a0d8-5b2e485006ea"}
{"level":"error","ts":"2026-06-11T10:25:02.383Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"4cb2af3f-936f-4ea4-98bb-3eeadb0e2302","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.383Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"4cb2af3f-936f-4ea4-98bb-3eeadb0e2302"}
{"level":"error","ts":"2026-06-11T10:25:02.383Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"4cb2af3f-936f-4ea4-98bb-3eeadb0e2302","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.392Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"33654d68-253d-48b7-bc8c-69e4b347d0d5"}
{"level":"error","ts":"2026-06-11T10:25:02.393Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"c237c879-00ee-4550-b6c9-87fb3a2a16c0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.393Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"c237c879-00ee-4550-b6c9-87fb3a2a16c0"}
{"level":"error","ts":"2026-06-11T10:25:02.393Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"c237c879-00ee-4550-b6c9-87fb3a2a16c0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"reconcileID":"8014943c-7428-4a9b-930e-b5471472eab1"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"14754663-ff81-4bcf-8183-0c865fc45323"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"71406841-60c5-4c20-9734-275ac821158c","rayCluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"0e885c10-450e-4bb2-b4b4-b4274102d2dd"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"71406841-60c5-4c20-9734-275ac821158c","pod":"stop-cn5qj-head-9ljzg","ip":"10.133.0.28"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"71406841-60c5-4c20-9734-275ac821158c","pod":"stop-cn5qj-small-group-worker-xvwbb","ip":"10.134.0.41"}
{"level":"info","ts":"2026-06-11T10:25:02.399Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"71406841-60c5-4c20-9734-275ac821158c","rayCluster":"stop-cn5qj","podIPs":["10.133.0.28","10.134.0.41"]}
{"level":"info","ts":"2026-06-11T10:25:02.408Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","name":"kuberay-gateway-access","namespace":"test-ns-c6srn"}
{"level":"error","ts":"2026-06-11T10:25:02.414Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"9b8ca125-ef64-4f4a-a970-e77b1c1ab7e8","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.414Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"9b8ca125-ef64-4f4a-a970-e77b1c1ab7e8"}
{"level":"error","ts":"2026-06-11T10:25:02.414Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"9b8ca125-ef64-4f4a-a970-e77b1c1ab7e8","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.416Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","name":"test-ns-c6srn-stop-cn5qj","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:25:02.424Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","configMap":"kube-rbac-proxy-config-stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.430Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","serviceAccount":"stop-cn5qj-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:25:02.452Z","logger":"raycluster-resource","msg":"default","name":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.452Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-cn5qj","namespace":"test-ns-c6srn"}
{"level":"error","ts":"2026-06-11T10:25:02.454Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"65a975f3-685c-44f3-bed2-a471b68fd6fa","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.454Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"65a975f3-685c-44f3-bed2-a471b68fd6fa"}
{"level":"error","ts":"2026-06-11T10:25:02.454Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"65a975f3-685c-44f3-bed2-a471b68fd6fa","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.463Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"403e2051-f747-49a4-9de2-307b21e4af72","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.463Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.463Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.463Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237","cluster":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.463Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-11T10:25:02.463Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-11T10:25:02.464Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"baca1ddc-8976-42f1-807a-ee5b4eb0fbcb"}
{"level":"info","ts":"2026-06-11T10:25:02.464Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4bd27238-124c-46d9-a662-a6fc336e41a9","namespace":"test-ns-c6srn","clusterName":"stop-cn5qj"}
{"level":"info","ts":"2026-06-11T10:25:02.464Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"6a2cd98d-0217-4db5-9865-f6568b2001c3"}
{"level":"error","ts":"2026-06-11T10:25:02.468Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237","error":"rayclusters.ray.io \"stop-cn5qj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.468Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237"}
{"level":"error","ts":"2026-06-11T10:25:02.468Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4d9944b6-1f52-4eba-aedd-34b4e0c68237","error":"rayclusters.ray.io \"stop-cn5qj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.468Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b5fbaac3-55f8-4b68-9c19-4333312a60ac","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.468Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b5fbaac3-55f8-4b68-9c19-4333312a60ac"}
{"level":"info","ts":"2026-06-11T10:25:02.468Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b5fbaac3-55f8-4b68-9c19-4333312a60ac","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:25:02.468Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"b5fbaac3-55f8-4b68-9c19-4333312a60ac","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:02.474Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"03c8e512-49bf-4d64-baec-e5e01f056697","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.474Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"03c8e512-49bf-4d64-baec-e5e01f056697"}
{"level":"info","ts":"2026-06-11T10:25:02.474Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"03c8e512-49bf-4d64-baec-e5e01f056697","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:25:02.474Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"03c8e512-49bf-4d64-baec-e5e01f056697","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:02.476Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9894435b-f6a8-43fb-8534-c8f4af8001b2","namespacedName":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:02.476Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9894435b-f6a8-43fb-8534-c8f4af8001b2"}
{"level":"info","ts":"2026-06-11T10:25:02.476Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9894435b-f6a8-43fb-8534-c8f4af8001b2","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:25:02.476Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"9894435b-f6a8-43fb-8534-c8f4af8001b2","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:02.490Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"ec9ef906-ba14-48c2-a10b-e60bfa2512a7"}
{"level":"info","ts":"2026-06-11T10:25:02.527Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"928a53c1-a111-47d6-b360-26d6e56be98b"}
{"level":"error","ts":"2026-06-11T10:25:02.535Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"78e9d1e8-e28f-4493-ba1b-f8a82b51175a","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.535Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"78e9d1e8-e28f-4493-ba1b-f8a82b51175a"}
{"level":"error","ts":"2026-06-11T10:25:02.535Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"78e9d1e8-e28f-4493-ba1b-f8a82b51175a","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.550Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-cn5qj","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"stop-cn5qj","reconcileID":"4bd27238-124c-46d9-a662-a6fc336e41a9","namespace":"test-ns-c6srn","clusterName":"stop-cn5qj"}
{"level":"error","ts":"2026-06-11T10:25:02.696Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"7d12255c-ef01-453f-a4ab-4f083301048e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:02.696Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"7d12255c-ef01-453f-a4ab-4f083301048e"}
{"level":"error","ts":"2026-06-11T10:25:02.696Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"7d12255c-ef01-453f-a4ab-4f083301048e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:03.017Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"f22caa34-5c61-4374-bd72-eb2c2c0892a8","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:03.017Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"f22caa34-5c61-4374-bd72-eb2c2c0892a8"}
{"level":"error","ts":"2026-06-11T10:25:03.017Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"f22caa34-5c61-4374-bd72-eb2c2c0892a8","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:03.658Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"a3a57dfe-a7d2-4db9-8e09-e81114ddd430","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:03.658Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"a3a57dfe-a7d2-4db9-8e09-e81114ddd430"}
{"level":"error","ts":"2026-06-11T10:25:03.658Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"a3a57dfe-a7d2-4db9-8e09-e81114ddd430","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:04.940Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"f6149dc3-0ca3-4812-9b81-5a8153b12828","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:04.940Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"f6149dc3-0ca3-4812-9b81-5a8153b12828"}
{"level":"error","ts":"2026-06-11T10:25:04.940Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"f6149dc3-0ca3-4812-9b81-5a8153b12828","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:05.045Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-c6srn"},"reconcileID":"7a9e8e77-3cfe-4942-be34-02287648fade"}
{"level":"info","ts":"2026-06-11T10:25:07.388Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6141dda1-e3c5-4200-9307-b910c0a7936f","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:07.389Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6141dda1-e3c5-4200-9307-b910c0a7936f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:25:07.404Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6141dda1-e3c5-4200-9307-b910c0a7936f"}
{"level":"info","ts":"2026-06-11T10:25:07.404Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6141dda1-e3c5-4200-9307-b910c0a7936f","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:25:07.404Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6141dda1-e3c5-4200-9307-b910c0a7936f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-hcqjg","rayClusterName":"long-running-qslmq","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:25:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:25:07.404Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6141dda1-e3c5-4200-9307-b910c0a7936f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:25:07.418Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:07.418Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21"}
{"level":"info","ts":"2026-06-11T10:25:07.418Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:25:07.418Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-6pfqp","rayClusterName":"long-running-cc6tx","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:25:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:25:07.418Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:25:07.428Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-11T10:25:07.428Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21"}
{"level":"error","ts":"2026-06-11T10:25:07.428Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"0b851976-ffe9-4f81-86d0-408c5f63ea21","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.428Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"d7c816d9-8352-4dfe-92a2-bc222fe12e08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:07.428Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"d7c816d9-8352-4dfe-92a2-bc222fe12e08","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.428Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"d7c816d9-8352-4dfe-92a2-bc222fe12e08","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.428Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"d7c816d9-8352-4dfe-92a2-bc222fe12e08","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.434Z","logger":"raycluster-resource","msg":"default","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.434Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qslmq","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c8ae81a3-7577-45b5-aa0c-a207e4278d8d","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"d7c816d9-8352-4dfe-92a2-bc222fe12e08","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"d7c816d9-8352-4dfe-92a2-bc222fe12e08"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"d7c816d9-8352-4dfe-92a2-bc222fe12e08","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"c7be9e35-a5bc-465f-9ed6-26ba2be4c230","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"c7be9e35-a5bc-465f-9ed6-26ba2be4c230","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.456Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"c7be9e35-a5bc-465f-9ed6-26ba2be4c230","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.457Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"c7be9e35-a5bc-465f-9ed6-26ba2be4c230"}
{"level":"info","ts":"2026-06-11T10:25:07.457Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"c7be9e35-a5bc-465f-9ed6-26ba2be4c230","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:07.464Z","logger":"raycluster-resource","msg":"default","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.464Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qslmq","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","name":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:25:07.466Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","name":"long-running-qslmq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:07.483Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c8ae81a3-7577-45b5-aa0c-a207e4278d8d","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.483Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"5864711f-5987-4a98-93fc-71c1714f3e4d","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.486Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"74ef3ea3-39cd-4fea-9d2a-9d14c47720a7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:07.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"74ef3ea3-39cd-4fea-9d2a-9d14c47720a7","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.486Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"74ef3ea3-39cd-4fea-9d2a-9d14c47720a7","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.486Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"74ef3ea3-39cd-4fea-9d2a-9d14c47720a7"}
{"level":"info","ts":"2026-06-11T10:25:07.486Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"74ef3ea3-39cd-4fea-9d2a-9d14c47720a7","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","name":"long-running-qslmq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"long-running-qslmq"}}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d"}
{"level":"error","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"feea3372-2b35-4fcf-af07-b9e07234af4d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16"}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16"}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:25:07.487Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","name":"long-running-qslmq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-06-11T10:25:07.527Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"55f74340-baae-45dc-b5d6-9d83997b8c45","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.527Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"55f74340-baae-45dc-b5d6-9d83997b8c45"}
{"level":"error","ts":"2026-06-11T10:25:07.527Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"55f74340-baae-45dc-b5d6-9d83997b8c45","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.527Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","name":"long-running-qslmq-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:25:07.527Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16"}
{"level":"error","ts":"2026-06-11T10:25:07.527Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"eee9f4c3-4187-40a0-9198-7ee303732b16","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","name":"long-running-qslmq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"4b5a2dee-5dba-4c04-b6f8-b0f1bbaa43ab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"4b5a2dee-5dba-4c04-b6f8-b0f1bbaa43ab","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"4b5a2dee-5dba-4c04-b6f8-b0f1bbaa43ab","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"4b5a2dee-5dba-4c04-b6f8-b0f1bbaa43ab"}
{"level":"info","ts":"2026-06-11T10:25:07.528Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"4b5a2dee-5dba-4c04-b6f8-b0f1bbaa43ab","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:07.536Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"5864711f-5987-4a98-93fc-71c1714f3e4d","name":"long-running-qslmq-head"}
{"level":"info","ts":"2026-06-11T10:25:07.540Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","name":"kuberay-gateway-access","namespace":"test-ns-c6srn","operation":"created"}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","name":"long-running-qslmq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"long-running-qslmq"}}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743"}
{"level":"error","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b7db5dd2-ee78-45ed-9a13-613d16820743","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7"}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7"}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7"}
{"level":"error","ts":"2026-06-11T10:25:07.542Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b630216e-c0d0-4308-86b0-2a4b3e2701c7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.547Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a"}
{"level":"info","ts":"2026-06-11T10:25:07.547Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq","secret":"ca-secret"}
{"level":"info","ts":"2026-06-11T10:25:07.547Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.550Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","name":"test-ns-c6srn-long-running-qslmq","operation":"created"}
{"level":"info","ts":"2026-06-11T10:25:07.553Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"5864711f-5987-4a98-93fc-71c1714f3e4d","name":"long-running-qslmq-workers"}
{"level":"info","ts":"2026-06-11T10:25:07.553Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"5864711f-5987-4a98-93fc-71c1714f3e4d","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.553Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"95eadc28-1686-4a21-b3b9-d753ca6495b1","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.560Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","name":"kube-rbac-proxy-config-long-running-qslmq","operation":"created"}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913"}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913"}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913"}
{"level":"error","ts":"2026-06-11T10:25:07.563Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"137773ad-3ba0-44bb-9faa-d932c38bf913","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:07.627Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.627Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc"}
{"level":"error","ts":"2026-06-11T10:25:07.627Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8762361d-4cfb-4847-9a2b-0561f31cb6fc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.627Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"009b4aa9-8c80-40c2-9d42-f8f42ff13cb6","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.627Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"009b4aa9-8c80-40c2-9d42-f8f42ff13cb6","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.627Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"009b4aa9-8c80-40c2-9d42-f8f42ff13cb6","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.627Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.636Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"95eadc28-1686-4a21-b3b9-d753ca6495b1","name":"long-running-qslmq-head"}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d"}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d"}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d"}
{"level":"error","ts":"2026-06-11T10:25:07.644Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"df266bd5-d3b3-45c9-985c-c0bea29d724d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.651Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a"}
{"level":"info","ts":"2026-06-11T10:25:07.651Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:25:07.651Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"009b4aa9-8c80-40c2-9d42-f8f42ff13cb6","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.651Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"24317bc8-c56d-49dd-988a-a37708d52052","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.651Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"24317bc8-c56d-49dd-988a-a37708d52052","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.651Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"24317bc8-c56d-49dd-988a-a37708d52052","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.652Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5a47ec74-599e-4092-b01c-5b5878e8cf08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:07.652Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5a47ec74-599e-4092-b01c-5b5878e8cf08","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.652Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5a47ec74-599e-4092-b01c-5b5878e8cf08","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5a47ec74-599e-4092-b01c-5b5878e8cf08"}
{"level":"info","ts":"2026-06-11T10:25:07.652Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5a47ec74-599e-4092-b01c-5b5878e8cf08","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:07.657Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"95eadc28-1686-4a21-b3b9-d753ca6495b1","name":"long-running-qslmq-workers"}
{"level":"info","ts":"2026-06-11T10:25:07.657Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"95eadc28-1686-4a21-b3b9-d753ca6495b1","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.657Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"9434da14-2512-4a04-bada-190b45941e22","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:07.660Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"24317bc8-c56d-49dd-988a-a37708d52052","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.660Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"24317bc8-c56d-49dd-988a-a37708d52052"}
{"level":"error","ts":"2026-06-11T10:25:07.660Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"24317bc8-c56d-49dd-988a-a37708d52052","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.664Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a"}
{"level":"info","ts":"2026-06-11T10:25:07.664Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","rayCluster":"long-running-qslmq","podIPs":[]}
{"level":"error","ts":"2026-06-11T10:25:07.664Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qslmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.664Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a"}
{"level":"error","ts":"2026-06-11T10:25:07.664Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ebaa1caf-3632-4b8e-bc43-0321d502ab1a","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-qslmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.664Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6df3cebc-52e3-4cc4-8e81-58154e95c7a4","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.664Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6df3cebc-52e3-4cc4-8e81-58154e95c7a4"}
{"level":"info","ts":"2026-06-11T10:25:07.665Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"57a96087-5022-4d0c-b76f-ebd27cd6bda0","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:07.665Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"57a96087-5022-4d0c-b76f-ebd27cd6bda0","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.665Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"57a96087-5022-4d0c-b76f-ebd27cd6bda0","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.670Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2fa28c62-a027-47bd-9d7b-7864ef3b5eae","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.670Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2fa28c62-a027-47bd-9d7b-7864ef3b5eae"}
{"level":"info","ts":"2026-06-11T10:25:07.727Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"57a96087-5022-4d0c-b76f-ebd27cd6bda0","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.732Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"9434da14-2512-4a04-bada-190b45941e22","name":"long-running-qslmq-head"}
{"level":"info","ts":"2026-06-11T10:25:07.748Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"9434da14-2512-4a04-bada-190b45941e22","name":"long-running-qslmq-workers"}
{"level":"info","ts":"2026-06-11T10:25:07.748Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"9434da14-2512-4a04-bada-190b45941e22","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb"}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb"}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:07.804Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb"}
{"level":"error","ts":"2026-06-11T10:25:07.805Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"a64b1563-68fb-4373-901d-ac989c5031bb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d"}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d"}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d"}
{"level":"error","ts":"2026-06-11T10:25:08.125Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3fb349eb-680d-4032-98ff-1375a4ccde1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992"}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992"}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","error":"head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"error","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found"}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992"}
{"level":"error","ts":"2026-06-11T10:25:08.766Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"79e7938c-87ce-4526-bdb8-b99b60a06992","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-qslmq not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","cluster":"long-running-qslmq","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","cluster":"long-running-qslmq","authType":"IntegratedOAuth","serviceAccount":"long-running-qslmq-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-qslmq","ray.io/group":"headgroup","ray.io/identifier":"long-running-qslmq-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:25:10.048Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","enabled":true}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","name":"long-running-qslmq-head-ktnqb"}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c"}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:10.068Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","enabled":true}
{"level":"info","ts":"2026-06-11T10:25:10.083Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","name":"long-running-qslmq-small-group-worker-7sw74"}
{"level":"info","ts":"2026-06-11T10:25:10.083Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.083Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-qslmq-head-svc"},"newHead":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"}}
{"level":"info","ts":"2026-06-11T10:25:10.083Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0a9d5805-21cf-42c2-8b17-eafe09e43e6c","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2f6beac0-87f5-4734-a3f1-fe2b2e714128","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0c671952-88d0-4db6-9845-1464e0ba5ad5","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"efd8a47c-ca80-4f37-98b6-eebc830cc9a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"efd8a47c-ca80-4f37-98b6-eebc830cc9a6","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2f6beac0-87f5-4734-a3f1-fe2b2e714128","rayCluster":"long-running-qslmq","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"efd8a47c-ca80-4f37-98b6-eebc830cc9a6","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-qslmq-head-svc"},"newHead":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"}}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"efd8a47c-ca80-4f37-98b6-eebc830cc9a6"}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"efd8a47c-ca80-4f37-98b6-eebc830cc9a6","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:10.100Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:10.110Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","name":"long-running-qslmq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-c6srn","name":"long-running-qslmq"}}
{"level":"info","ts":"2026-06-11T10:25:10.110Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d"}
{"level":"error","ts":"2026-06-11T10:25:10.110Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0e9b5242-da11-4462-b0a7-1990c5d4f60d","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-qslmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:10.115Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"75acc475-e747-49a2-a5ad-8b41803c1f51","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d"}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d"}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:10.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.117Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.117Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.117Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:10.117Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d2ee0eb3-89e2-4968-8902-6cbdab50de2d","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:10.127Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0c671952-88d0-4db6-9845-1464e0ba5ad5","name":"long-running-qslmq-head"}
{"level":"info","ts":"2026-06-11T10:25:10.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b"}
{"level":"info","ts":"2026-06-11T10:25:10.127Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b"}
{"level":"info","ts":"2026-06-11T10:25:10.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:10.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:25:10.128Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1a9701bb-a2c3-456b-af14-e74f02abb24b","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:10.143Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"cea68ae3-a77b-4d35-bb6b-2e6871a905b3","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.143Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"cea68ae3-a77b-4d35-bb6b-2e6871a905b3","rayCluster":"long-running-qslmq","podIPs":[]}
{"level":"info","ts":"2026-06-11T10:25:10.146Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5e8073f1-d760-4f17-9e0b-57fbfb4cd580","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:10.146Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5e8073f1-d760-4f17-9e0b-57fbfb4cd580","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.146Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5e8073f1-d760-4f17-9e0b-57fbfb4cd580","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:10.146Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5e8073f1-d760-4f17-9e0b-57fbfb4cd580"}
{"level":"info","ts":"2026-06-11T10:25:10.146Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"5e8073f1-d760-4f17-9e0b-57fbfb4cd580","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:10.228Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0c671952-88d0-4db6-9845-1464e0ba5ad5","name":"long-running-qslmq-workers"}
{"level":"info","ts":"2026-06-11T10:25:10.228Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455"}
{"level":"info","ts":"2026-06-11T10:25:10.327Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0c671952-88d0-4db6-9845-1464e0ba5ad5","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.327Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2763f29f-d293-4e36-98ba-19eea378d82f","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","podName":"long-running-qslmq-small-group-worker-7sw74","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","podName":"long-running-qslmq-small-group-worker-7sw74","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:10.328Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"db0d5140-b3d3-4b96-a466-8d0b3a297455","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:10.342Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2763f29f-d293-4e36-98ba-19eea378d82f","name":"long-running-qslmq-head"}
{"level":"info","ts":"2026-06-11T10:25:10.355Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2763f29f-d293-4e36-98ba-19eea378d82f","name":"long-running-qslmq-workers"}
{"level":"info","ts":"2026-06-11T10:25:10.355Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2763f29f-d293-4e36-98ba-19eea378d82f","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.457Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"33337445-95f9-4e0f-a734-78244cb0ecdb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:10.457Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"33337445-95f9-4e0f-a734-78244cb0ecdb","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:10.457Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"33337445-95f9-4e0f-a734-78244cb0ecdb","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:10.457Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"33337445-95f9-4e0f-a734-78244cb0ecdb"}
{"level":"info","ts":"2026-06-11T10:25:10.457Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"33337445-95f9-4e0f-a734-78244cb0ecdb","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7"}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7"}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.534Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.535Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:10.535Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"392a6358-c2c4-42c5-994a-40148901e1b7","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:10.547Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9a03e69d-4f7c-456f-b2e3-bddb5063cf5f","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:11.149Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a"}
{"level":"info","ts":"2026-06-11T10:25:11.149Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a"}
{"level":"info","ts":"2026-06-11T10:25:11.149Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:11.150Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"cda8ed6c-6c9b-451e-8074-1b8acaace09a","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:11.506Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755"}
{"level":"info","ts":"2026-06-11T10:25:11.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755"}
{"level":"info","ts":"2026-06-11T10:25:11.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:11.506Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"newHead":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"}}
{"level":"info","ts":"2026-06-11T10:25:11.507Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"610b6f7f-3142-4193-8bad-1543c9169755","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"d3ddc130-6f2f-4037-87a6-09db6fe97061","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c488d5fc-22d9-4e8c-a18e-491e70f61473","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"d3ddc130-6f2f-4037-87a6-09db6fe97061","pod":"long-running-qslmq-head-ktnqb","ip":"10.133.0.29"}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"d3ddc130-6f2f-4037-87a6-09db6fe97061","pod":"long-running-qslmq-small-group-worker-7sw74","ip":"10.134.0.44"}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3ceebb7b-1f0d-44ca-9017-473e8cc42b75","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3ceebb7b-1f0d-44ca-9017-473e8cc42b75","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3ceebb7b-1f0d-44ca-9017-473e8cc42b75","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3ceebb7b-1f0d-44ca-9017-473e8cc42b75"}
{"level":"info","ts":"2026-06-11T10:25:11.527Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3ceebb7b-1f0d-44ca-9017-473e8cc42b75","RayCluster":"long-running-qslmq","State":""}
{"level":"info","ts":"2026-06-11T10:25:11.539Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"d3ddc130-6f2f-4037-87a6-09db6fe97061","certificate":"ray-head-cert-long-running-qslmq","dnsNames":["long-running-qslmq-head-svc","localhost","long-running-qslmq-head-svc.test-ns-c6srn.svc","long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.29","10.134.0.44","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:25:11.552Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c488d5fc-22d9-4e8c-a18e-491e70f61473","name":"long-running-qslmq-head"}
{"level":"info","ts":"2026-06-11T10:25:11.566Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"d3ddc130-6f2f-4037-87a6-09db6fe97061","certificate":"ray-worker-cert-long-running-qslmq","dnsNames":["long-running-qslmq-worker-svc","localhost","long-running-qslmq-small-group","long-running-qslmq-worker-svc.test-ns-c6srn.svc","long-running-qslmq-worker-svc.test-ns-c6srn.svc.cluster.local","*.long-running-qslmq-worker-svc.test-ns-c6srn.svc","*.long-running-qslmq-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.29","10.134.0.44","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:25:11.566Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"d3ddc130-6f2f-4037-87a6-09db6fe97061","rayCluster":"long-running-qslmq","podIPs":["10.133.0.29","10.134.0.44"]}
{"level":"info","ts":"2026-06-11T10:25:11.577Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c488d5fc-22d9-4e8c-a18e-491e70f61473","name":"long-running-qslmq-workers"}
{"level":"info","ts":"2026-06-11T10:25:11.577Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c488d5fc-22d9-4e8c-a18e-491e70f61473","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:12.144Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6bd1e00-f370-4237-afe5-462541ffeb46","seconds":300}
{"level":"error","ts":"2026-06-11T10:25:12.648Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"469fd9c7-0ac1-40a3-bfd2-5be51e3ce84d","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:12.648Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"469fd9c7-0ac1-40a3-bfd2-5be51e3ce84d"}
{"level":"error","ts":"2026-06-11T10:25:12.648Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"469fd9c7-0ac1-40a3-bfd2-5be51e3ce84d","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:13.458Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6be56817-617e-47dc-a026-8b0a82d1c6ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:13.458Z","logger":"controllers.RayJob","msg":"The RayJob has passed the activeDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6be56817-617e-47dc-a026-8b0a82d1c6ff","StartTime":"2026-06-11 10:25:07 +0000 UTC","ActiveDeadlineSeconds":5}
{"level":"info","ts":"2026-06-11T10:25:13.458Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6be56817-617e-47dc-a026-8b0a82d1c6ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-hcqjg","rayClusterName":"long-running-qslmq","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:25:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-hcqjg","rayClusterName":"long-running-qslmq","jobDeploymentStatus":"Failed","reason":"DeadlineExceeded","message":"The RayJob has passed the activeDeadlineSeconds. StartTime: 2026-06-11 10:25:07 +0000 UTC. ActiveDeadlineSeconds: 5","startTime":"2026-06-11T10:25:07Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:25:13.458Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6be56817-617e-47dc-a026-8b0a82d1c6ff","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-11T10:25:13.479Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"281cf874-afd5-4da9-9370-bda2cc0006a7","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:13.479Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"281cf874-afd5-4da9-9370-bda2cc0006a7","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:13.479Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:13.479Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"281cf874-afd5-4da9-9370-bda2cc0006a7","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:13.479Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"281cf874-afd5-4da9-9370-bda2cc0006a7","seconds":601}
{"level":"info","ts":"2026-06-11T10:25:14.465Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-c6srn"},"reconcileID":"c7eb2122-d464-4f51-a28f-4206037ae4d5","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-06-11T10:25:14.512Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"managed-externally","namespace":"test-ns-c6srn"},"reconcileID":"9ec570fb-7bc5-407d-93a8-61e110e9fd5d"}
{"level":"info","ts":"2026-06-11T10:25:14.689Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-5fsk2"},"reconcileID":"8cf0f50e-e5ad-49ac-83a0-5f6060953d21","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-06-11T10:25:16.480Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3f855268-2ee0-4017-b203-99ee3aed3d0d","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:16.480Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3f855268-2ee0-4017-b203-99ee3aed3d0d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:16.480Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:16.480Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3f855268-2ee0-4017-b203-99ee3aed3d0d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:16.480Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"3f855268-2ee0-4017-b203-99ee3aed3d0d","seconds":598}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:16.531Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"c70ed4ab-b110-467c-87af-53b7d6fe651c","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:16.590Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c16e58ed-b407-4a19-b0fe-8bfba9c984f7","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:16.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c16e58ed-b407-4a19-b0fe-8bfba9c984f7","pod":"raycluster-head-t5q9r","ip":"10.132.0.27"}
{"level":"info","ts":"2026-06-11T10:25:16.590Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c16e58ed-b407-4a19-b0fe-8bfba9c984f7","pod":"raycluster-small-group-worker-n98tf","ip":"10.134.0.25"}
{"level":"info","ts":"2026-06-11T10:25:16.590Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"},"namespace":"test-ns-5fsk2","name":"raycluster","reconcileID":"c16e58ed-b407-4a19-b0fe-8bfba9c984f7","rayCluster":"raycluster","podIPs":["10.132.0.27","10.134.0.25"]}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8"}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8"}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-11T10:25:17.551Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"126a2c70-7efa-4367-8a24-23725d8ac8f8","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2a764cf6-c7ff-4f54-8873-2e4201591bd8","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3c86e952-2c23-4823-bc0b-5ecef6f25c3f","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2a764cf6-c7ff-4f54-8873-2e4201591bd8","pod":"long-running-qslmq-head-ktnqb","ip":"10.133.0.29"}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2a764cf6-c7ff-4f54-8873-2e4201591bd8","pod":"long-running-qslmq-small-group-worker-7sw74","ip":"10.134.0.44"}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"345a0de7-5a91-4c4f-9de3-62d0050a4b69","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2a764cf6-c7ff-4f54-8873-2e4201591bd8","rayCluster":"long-running-qslmq","podIPs":["10.133.0.29","10.134.0.44"]}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"345a0de7-5a91-4c4f-9de3-62d0050a4b69","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:17.575Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"345a0de7-5a91-4c4f-9de3-62d0050a4b69","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:17.575Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"345a0de7-5a91-4c4f-9de3-62d0050a4b69","seconds":597}
{"level":"info","ts":"2026-06-11T10:25:17.585Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3c86e952-2c23-4823-bc0b-5ecef6f25c3f","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:17.716Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"288d450d-d6c3-49e6-8513-37e7c29fadc7","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.716Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"288d450d-d6c3-49e6-8513-37e7c29fadc7","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:25:17.729Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"288d450d-d6c3-49e6-8513-37e7c29fadc7"}
{"level":"info","ts":"2026-06-11T10:25:17.729Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"288d450d-d6c3-49e6-8513-37e7c29fadc7","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:25:17.729Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"288d450d-d6c3-49e6-8513-37e7c29fadc7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdcbj","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:25:17.729Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"288d450d-d6c3-49e6-8513-37e7c29fadc7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:25:17.740Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.740Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.740Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:17.740Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","head service name":"raycluster-head-svc","namespace":"test-ns-5fsk2"}
{"level":"info","ts":"2026-06-11T10:25:17.740Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","head service URL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:25:17.740Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b"}
{"level":"info","ts":"2026-06-11T10:25:17.740Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","counter-kdcbj",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-kdcbj","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","--follow","counter-kdcbj"]}
{"level":"info","ts":"2026-06-11T10:25:17.755Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-11T10:25:17.755Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.755Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdcbj","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:25:17.755Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"435080c8-4709-4cd1-847d-22e69d4aee7b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:25:17.773Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ca5410b6-bd7f-43cd-82bc-f6b92ce30279","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.773Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ca5410b6-bd7f-43cd-82bc-f6b92ce30279","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.773Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ca5410b6-bd7f-43cd-82bc-f6b92ce30279","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:17.783Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ca5410b6-bd7f-43cd-82bc-f6b92ce30279","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.783Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ca5410b6-bd7f-43cd-82bc-f6b92ce30279"}
{"level":"error","ts":"2026-06-11T10:25:17.783Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ca5410b6-bd7f-43cd-82bc-f6b92ce30279","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"5ddf7fe4-d07f-42fe-8966-798f28f822b7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.789Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"5ddf7fe4-d07f-42fe-8966-798f28f822b7","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.789Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"5ddf7fe4-d07f-42fe-8966-798f28f822b7","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:17.794Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"5ddf7fe4-d07f-42fe-8966-798f28f822b7","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.794Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"5ddf7fe4-d07f-42fe-8966-798f28f822b7"}
{"level":"error","ts":"2026-06-11T10:25:17.794Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"5ddf7fe4-d07f-42fe-8966-798f28f822b7","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"64707363-0f23-4c79-aa36-57a3b7561bc7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.794Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"64707363-0f23-4c79-aa36-57a3b7561bc7","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.794Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"64707363-0f23-4c79-aa36-57a3b7561bc7","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:17.798Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"64707363-0f23-4c79-aa36-57a3b7561bc7","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.798Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"64707363-0f23-4c79-aa36-57a3b7561bc7"}
{"level":"error","ts":"2026-06-11T10:25:17.798Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"64707363-0f23-4c79-aa36-57a3b7561bc7","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.804Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"1fc9064b-a751-43af-9a45-4d9bef5b7280","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.804Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"1fc9064b-a751-43af-9a45-4d9bef5b7280","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.804Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"1fc9064b-a751-43af-9a45-4d9bef5b7280","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:17.809Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"1fc9064b-a751-43af-9a45-4d9bef5b7280","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.809Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"1fc9064b-a751-43af-9a45-4d9bef5b7280"}
{"level":"error","ts":"2026-06-11T10:25:17.809Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"1fc9064b-a751-43af-9a45-4d9bef5b7280","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fa9d8fec-c2a0-4b5c-9087-5f7c466e61d7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fa9d8fec-c2a0-4b5c-9087-5f7c466e61d7","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fa9d8fec-c2a0-4b5c-9087-5f7c466e61d7","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:17.855Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fa9d8fec-c2a0-4b5c-9087-5f7c466e61d7","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.855Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fa9d8fec-c2a0-4b5c-9087-5f7c466e61d7"}
{"level":"error","ts":"2026-06-11T10:25:17.855Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fa9d8fec-c2a0-4b5c-9087-5f7c466e61d7","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"cd31af65-bff3-49ca-9002-afcdf5953380","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:17.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"cd31af65-bff3-49ca-9002-afcdf5953380","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:17.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"cd31af65-bff3-49ca-9002-afcdf5953380","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:17.942Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"cd31af65-bff3-49ca-9002-afcdf5953380","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:17.942Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"cd31af65-bff3-49ca-9002-afcdf5953380"}
{"level":"error","ts":"2026-06-11T10:25:17.942Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"cd31af65-bff3-49ca-9002-afcdf5953380","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:18.102Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c742bae7-1dc4-4846-bca8-ed57e3fe23c7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:18.102Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c742bae7-1dc4-4846-bca8-ed57e3fe23c7","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:18.102Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c742bae7-1dc4-4846-bca8-ed57e3fe23c7","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:18.108Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c742bae7-1dc4-4846-bca8-ed57e3fe23c7","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:18.108Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c742bae7-1dc4-4846-bca8-ed57e3fe23c7"}
{"level":"error","ts":"2026-06-11T10:25:18.108Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c742bae7-1dc4-4846-bca8-ed57e3fe23c7","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:18.429Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"04e357ae-d03c-42cd-bd03-a2c00059cd53","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:18.429Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"04e357ae-d03c-42cd-bd03-a2c00059cd53","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:18.429Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"04e357ae-d03c-42cd-bd03-a2c00059cd53","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:18.435Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"04e357ae-d03c-42cd-bd03-a2c00059cd53","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:18.435Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"04e357ae-d03c-42cd-bd03-a2c00059cd53"}
{"level":"error","ts":"2026-06-11T10:25:18.435Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"04e357ae-d03c-42cd-bd03-a2c00059cd53","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:19.076Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"e8c2baef-b35c-415c-99db-4a5eb103ae2f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:19.076Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"e8c2baef-b35c-415c-99db-4a5eb103ae2f","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:19.076Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"e8c2baef-b35c-415c-99db-4a5eb103ae2f","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:19.081Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"e8c2baef-b35c-415c-99db-4a5eb103ae2f","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:19.081Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"e8c2baef-b35c-415c-99db-4a5eb103ae2f"}
{"level":"error","ts":"2026-06-11T10:25:19.081Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"e8c2baef-b35c-415c-99db-4a5eb103ae2f","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:19.576Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972"}
{"level":"info","ts":"2026-06-11T10:25:19.576Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972"}
{"level":"info","ts":"2026-06-11T10:25:19.576Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:19.577Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"9f700e1e-ddc3-4dc0-bcfb-135e1102e972","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88"}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88"}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.733Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.733Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:19.733Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ecc57b20-799b-4a7c-b800-74547c7b9c88","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f"}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f"}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:19.752Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"40002929-fd22-4981-8741-799db22d8a9f","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f"}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f"}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:19.792Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"d6311378-7cff-4374-908f-865c9c836b9f","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f"}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f"}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:19.805Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"8f9a698b-a128-4d0d-9d15-e1a084456f0f","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-qslmq-small-group-worker-7sw74. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","podName":"long-running-qslmq-small-group-worker-7sw74","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","podName":"long-running-qslmq-small-group-worker-7sw74","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:19.928Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ba2b2c0-86db-4c11-afe7-d9ec436ed72f","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:20.195Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"18f1097b-5ce3-4683-b7f9-255210c50dfb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:20.195Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"18f1097b-5ce3-4683-b7f9-255210c50dfb","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:20.196Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"18f1097b-5ce3-4683-b7f9-255210c50dfb","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:20.201Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"18f1097b-5ce3-4683-b7f9-255210c50dfb","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.201Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"18f1097b-5ce3-4683-b7f9-255210c50dfb"}
{"level":"error","ts":"2026-06-11T10:25:20.201Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"18f1097b-5ce3-4683-b7f9-255210c50dfb","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.216Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032"}
{"level":"info","ts":"2026-06-11T10:25:20.216Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032"}
{"level":"info","ts":"2026-06-11T10:25:20.216Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","worker Pod":"long-running-qslmq-small-group-worker-7sw74","shouldDelete":true,"reason":"The worker Pod long-running-qslmq-small-group-worker-7sw74 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-11T10:25:20.238Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.238Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-11T10:25:20.238Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","podName":"long-running-qslmq-small-group-worker-7sw74","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:25:20.238Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032"}
{"level":"error","ts":"2026-06-11T10:25:20.238Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"e0333de9-381f-4694-b69b-daadc6727032","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.243Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19"}
{"level":"info","ts":"2026-06-11T10:25:20.243Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19"}
{"level":"info","ts":"2026-06-11T10:25:20.243Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-11T10:25:20.244Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.244Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.244Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:20.244Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"2a230dff-b71c-460c-99a2-025fc1cfdd19","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:20.253Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab"}
{"level":"info","ts":"2026-06-11T10:25:20.253Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab"}
{"level":"info","ts":"2026-06-11T10:25:20.253Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.253Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-11T10:25:20.254Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.254Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.254Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-11T10:25:20.254Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4d889baa-2622-430f-8be6-b6cc7db376ab","seconds":300}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39"}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39"}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39"}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.257Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.266Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.266Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.267Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.267Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}]}
{"level":"info","ts":"2026-06-11T10:25:20.267Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7e1dfa03-bd04-4f97-b295-9578cfd32ea8","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c2b7bbd5-52f6-4475-bcd6-a2a19a96ab9b","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"f9ee477f-63fc-44a9-87ac-513b3e282bc2","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c2b7bbd5-52f6-4475-bcd6-a2a19a96ab9b","pod":"long-running-qslmq-head-ktnqb","ip":"10.133.0.29"}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"f9ee477f-63fc-44a9-87ac-513b3e282bc2","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:20.291Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"f9ee477f-63fc-44a9-87ac-513b3e282bc2","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"f9ee477f-63fc-44a9-87ac-513b3e282bc2","seconds":594}
{"level":"info","ts":"2026-06-11T10:25:20.291Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39"}
{"level":"error","ts":"2026-06-11T10:25:20.291Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"1ec3212e-0147-41e1-a2ae-2f7dd2ce1e39","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135"}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.297Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.302Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.302Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.302Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.302Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135"}
{"level":"error","ts":"2026-06-11T10:25:20.302Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"97fc6d08-20d2-4039-8683-7dd0f6824135","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.304Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7e1dfa03-bd04-4f97-b295-9578cfd32ea8","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:20.310Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c2b7bbd5-52f6-4475-bcd6-a2a19a96ab9b","certificate":"ray-head-cert-long-running-qslmq","dnsNames":["long-running-qslmq-head-svc","localhost","long-running-qslmq-head-svc.test-ns-c6srn.svc","long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.29","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e"}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e"}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e"}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.313Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.327Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.327Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.327Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.327Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e"}
{"level":"error","ts":"2026-06-11T10:25:20.327Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"ff973143-4438-422e-b70d-46545c23859e","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.348Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c"}
{"level":"info","ts":"2026-06-11T10:25:20.348Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c"}
{"level":"info","ts":"2026-06-11T10:25:20.348Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c"}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.349Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.353Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.353Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.353Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.353Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c"}
{"level":"error","ts":"2026-06-11T10:25:20.353Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"22c51308-72e0-4a42-ad8a-0a3de8d1c14c","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.355Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c2b7bbd5-52f6-4475-bcd6-a2a19a96ab9b","certificate":"ray-worker-cert-long-running-qslmq","dnsNames":["long-running-qslmq-worker-svc","localhost","long-running-qslmq-small-group","long-running-qslmq-worker-svc.test-ns-c6srn.svc","long-running-qslmq-worker-svc.test-ns-c6srn.svc.cluster.local","*.long-running-qslmq-worker-svc.test-ns-c6srn.svc","*.long-running-qslmq-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["10.133.0.29","127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:25:20.355Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c2b7bbd5-52f6-4475-bcd6-a2a19a96ab9b","rayCluster":"long-running-qslmq","podIPs":["10.133.0.29"]}
{"level":"info","ts":"2026-06-11T10:25:20.362Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"9b8d7651-157f-4ed4-a9cd-40bcaefadf44","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:20.362Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"9b8d7651-157f-4ed4-a9cd-40bcaefadf44","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:20.362Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"9b8d7651-157f-4ed4-a9cd-40bcaefadf44","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:20.367Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"9b8d7651-157f-4ed4-a9cd-40bcaefadf44","JobId":"counter-kdcbj","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.367Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"9b8d7651-157f-4ed4-a9cd-40bcaefadf44"}
{"level":"error","ts":"2026-06-11T10:25:20.367Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"9b8d7651-157f-4ed4-a9cd-40bcaefadf44","error":"Job counter-kdcbj does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6"}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.394Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.399Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.399Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.400Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6"}
{"level":"error","ts":"2026-06-11T10:25:20.400Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"aa6a9770-52f5-4f56-b01a-83b8c7ce44d6","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880"}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.480Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.485Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.486Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.486Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.486Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880"}
{"level":"error","ts":"2026-06-11T10:25:20.486Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"dde925c7-902d-47b7-8abc-329d7753f880","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.646Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6"}
{"level":"info","ts":"2026-06-11T10:25:20.646Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6"}
{"level":"info","ts":"2026-06-11T10:25:20.646Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6"}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.647Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.651Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.651Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.651Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.651Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6"}
{"level":"error","ts":"2026-06-11T10:25:20.651Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5e34868e-9464-48ac-84df-bd932ca62dc6","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75"}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:20.972Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:20.977Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:20.977Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:20.977Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:20.977Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75"}
{"level":"error","ts":"2026-06-11T10:25:20.977Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"3d01eaa8-9c13-4ee5-b8fe-bbf05400aa75","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896"}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:21.618Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:21.623Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:21.623Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:21.623Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:21.623Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896"}
{"level":"error","ts":"2026-06-11T10:25:21.623Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"b812984e-cf2d-4ad9-82c6-3ca052c72896","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:22.889Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"265a1b50-0a21-4382-8241-f9da114a41b7","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:22.889Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"265a1b50-0a21-4382-8241-f9da114a41b7"}
{"level":"error","ts":"2026-06-11T10:25:22.889Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"265a1b50-0a21-4382-8241-f9da114a41b7","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-qslmq-head-ktnqb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85"}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:22.903Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:22.907Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:22.907Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:22.908Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","podName":"long-running-qslmq-head-ktnqb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-11T10:25:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-11T10:25:22.908Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85"}
{"level":"error","ts":"2026-06-11T10:25:22.908Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"80070e6c-78a2-479e-8a21-8a923d783f85","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:23.714Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b"}
{"level":"info","ts":"2026-06-11T10:25:23.714Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b"}
{"level":"info","ts":"2026-06-11T10:25:23.714Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:23.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","Found 1 head Pod":"long-running-qslmq-head-ktnqb","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-11 10:25:16 +0000 UTC,FinishedAt:2026-06-11 10:25:23 +0000 UTC,ContainerID:cri-o://cce56db587c69066a396502084a530284cd4f234951ff0408a4ca92ac24d7786,}"}
{"level":"info","ts":"2026-06-11T10:25:23.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","head Pod":"long-running-qslmq-head-ktnqb","shouldDelete":true,"reason":"The head Pod long-running-qslmq-head-ktnqb status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-11T10:25:23.728Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod long-running-qslmq-head-ktnqb status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:23.728Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","error":"The head Pod long-running-qslmq-head-ktnqb status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-11T10:25:23.728Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:25:23.728Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}]}
{"level":"info","ts":"2026-06-11T10:25:23.728Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:23.759Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b"}
{"level":"error","ts":"2026-06-11T10:25:23.759Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"0ae70753-55d2-4c3c-adeb-6e7ffc77426b","error":"The head Pod long-running-qslmq-head-ktnqb status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b0c696e3-f386-4af1-9087-cb180f72b85a","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ac71ab6e-8e08-4597-bab5-99bb7e5c0374","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b0c696e3-f386-4af1-9087-cb180f72b85a","pod":"long-running-qslmq-head-ktnqb","ip":"10.133.0.29"}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"737aa7d1-2e3f-41a1-b4c1-ec7336681f6e","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"737aa7d1-2e3f-41a1-b4c1-ec7336681f6e","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:23.760Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"737aa7d1-2e3f-41a1-b4c1-ec7336681f6e","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"737aa7d1-2e3f-41a1-b4c1-ec7336681f6e","seconds":591}
{"level":"info","ts":"2026-06-11T10:25:23.760Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b0c696e3-f386-4af1-9087-cb180f72b85a"}
{"level":"info","ts":"2026-06-11T10:25:23.766Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ac71ab6e-8e08-4597-bab5-99bb7e5c0374","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9"}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","nodeType":"worker","rayStartParams":{"address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","rayStartCmd":"ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-11T10:25:24.576Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:24.580Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:24.580Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:24.580Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","podName":"long-running-qslmq-head-ktnqb","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-11T10:25:24.580Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9"}
{"level":"error","ts":"2026-06-11T10:25:24.580Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"5c0fb1d5-2973-4193-adbc-d42d96a6e2f9","error":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","cluster":"long-running-qslmq","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","cluster":"long-running-qslmq","authType":"IntegratedOAuth","serviceAccount":"long-running-qslmq-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-qslmq","ray.io/group":"headgroup","ray.io/identifier":"long-running-qslmq-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-11T10:25:24.582Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","enabled":true}
{"level":"error","ts":"2026-06-11T10:25:24.586Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-qslmq-head-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:24.586Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","error":"FailedCreateHeadPod\npods \"long-running-qslmq-head-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:24.586Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:24.586Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:24.586Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:24.586Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"newHead":{"serviceName":"long-running-qslmq-head-svc"}}
{"level":"info","ts":"2026-06-11T10:25:24.586Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.29","serviceIP":"10.133.0.29","podName":"long-running-qslmq-head-ktnqb","serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-qslmq-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-qslmq-head-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-11T10:25:24.605Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf"}
{"level":"error","ts":"2026-06-11T10:25:24.605Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"00a9f231-5ea8-4f56-a4fd-f691246911bf","error":"FailedCreateHeadPod\npods \"long-running-qslmq-head-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:24.607Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"43505643-95d1-4556-892d-f48bb143b94e","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:24.607Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2cbee3aa-24d7-496a-b7e8-70a03a163ca9","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:24.607Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"938e457d-39a7-42d3-9fa9-4a3c63b907b8","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:24.607Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"938e457d-39a7-42d3-9fa9-4a3c63b907b8","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:24.607Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:24.607Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"938e457d-39a7-42d3-9fa9-4a3c63b907b8","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:24.607Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"938e457d-39a7-42d3-9fa9-4a3c63b907b8","seconds":590}
{"level":"info","ts":"2026-06-11T10:25:24.618Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2cbee3aa-24d7-496a-b7e8-70a03a163ca9","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:24.631Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"43505643-95d1-4556-892d-f48bb143b94e","certificate":"ray-head-cert-long-running-qslmq","dnsNames":["long-running-qslmq-head-svc","localhost","long-running-qslmq-head-svc.test-ns-c6srn.svc","long-running-qslmq-head-svc.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:25:24.650Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"43505643-95d1-4556-892d-f48bb143b94e","certificate":"ray-worker-cert-long-running-qslmq","dnsNames":["long-running-qslmq-worker-svc","localhost","long-running-qslmq-small-group","long-running-qslmq-worker-svc.test-ns-c6srn.svc","long-running-qslmq-worker-svc.test-ns-c6srn.svc.cluster.local","*.long-running-qslmq-worker-svc.test-ns-c6srn.svc","*.long-running-qslmq-worker-svc.test-ns-c6srn.svc.cluster.local","*-worker-*.test-ns-c6srn.svc","*-worker-*.test-ns-c6srn.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-11T10:25:24.650Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"43505643-95d1-4556-892d-f48bb143b94e"}
{"level":"info","ts":"2026-06-11T10:25:25.185Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3fc097e2-a18b-4d77-9927-3d531831aeb3","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.185Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3fc097e2-a18b-4d77-9927-3d531831aeb3","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.185Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3fc097e2-a18b-4d77-9927-3d531831aeb3","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.218Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8b7c24b8-248f-4e19-80f6-36e597748928","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.218Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"08881205-1c01-4afb-8356-939b468c0c58","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:25.218Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"08881205-1c01-4afb-8356-939b468c0c58","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:25.218Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:25.218Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"08881205-1c01-4afb-8356-939b468c0c58","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:25.218Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"08881205-1c01-4afb-8356-939b468c0c58","seconds":589}
{"level":"info","ts":"2026-06-11T10:25:25.218Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"885d9cec-a5ae-4e21-b0ce-6c47f64840ec","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.219Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"885d9cec-a5ae-4e21-b0ce-6c47f64840ec"}
{"level":"error","ts":"2026-06-11T10:25:25.219Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3fc097e2-a18b-4d77-9927-3d531831aeb3","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.219Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3fc097e2-a18b-4d77-9927-3d531831aeb3"}
{"level":"error","ts":"2026-06-11T10:25:25.219Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"3fc097e2-a18b-4d77-9927-3d531831aeb3","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.225Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8b7c24b8-248f-4e19-80f6-36e597748928","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.225Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b502d62f-37e0-4fb8-8e96-c1cb4d086280","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.225Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b502d62f-37e0-4fb8-8e96-c1cb4d086280","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.225Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b502d62f-37e0-4fb8-8e96-c1cb4d086280","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.241Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b502d62f-37e0-4fb8-8e96-c1cb4d086280","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.241Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b502d62f-37e0-4fb8-8e96-c1cb4d086280"}
{"level":"error","ts":"2026-06-11T10:25:25.241Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b502d62f-37e0-4fb8-8e96-c1cb4d086280","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.251Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"703c7a06-320e-43e2-afa4-b60e3b10574e","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.251Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"703c7a06-320e-43e2-afa4-b60e3b10574e","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.251Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"703c7a06-320e-43e2-afa4-b60e3b10574e","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.269Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"703c7a06-320e-43e2-afa4-b60e3b10574e","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.269Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"703c7a06-320e-43e2-afa4-b60e3b10574e"}
{"level":"error","ts":"2026-06-11T10:25:25.269Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"703c7a06-320e-43e2-afa4-b60e3b10574e","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.289Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f3d92bb7-31b0-4b09-ac3e-53d54d51a942","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.289Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f3d92bb7-31b0-4b09-ac3e-53d54d51a942","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.289Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f3d92bb7-31b0-4b09-ac3e-53d54d51a942","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.306Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f3d92bb7-31b0-4b09-ac3e-53d54d51a942","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.306Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f3d92bb7-31b0-4b09-ac3e-53d54d51a942"}
{"level":"error","ts":"2026-06-11T10:25:25.306Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f3d92bb7-31b0-4b09-ac3e-53d54d51a942","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.347Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b25e1187-e4c8-4b98-a53d-dcdd8c4e7d76","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.347Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b25e1187-e4c8-4b98-a53d-dcdd8c4e7d76","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.347Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b25e1187-e4c8-4b98-a53d-dcdd8c4e7d76","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.368Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b25e1187-e4c8-4b98-a53d-dcdd8c4e7d76","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.368Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b25e1187-e4c8-4b98-a53d-dcdd8c4e7d76"}
{"level":"error","ts":"2026-06-11T10:25:25.368Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"b25e1187-e4c8-4b98-a53d-dcdd8c4e7d76","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.448Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"47bafafe-9a62-4870-88b0-85e113d8c5ad","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.448Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"47bafafe-9a62-4870-88b0-85e113d8c5ad","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.448Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"47bafafe-9a62-4870-88b0-85e113d8c5ad","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4"}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4"}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","1 head service found":"long-running-qslmq-head-svc"}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4"}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"Waiting for authentication resources to be ready","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","condition":"&Condition{Type:AuthenticationReady,Status:False,ObservedGeneration:1,LastTransitionTime:2026-06-11 10:25:25 +0000 UTC,Reason:AuthenticationFailed,Message:Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated,}","reason":"AuthenticationFailed","conditionStale":false,"observedGeneration":1,"currentGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"error","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","filter labels":["test-ns-c6srn",{"ray.io/cluster":"long-running-qslmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:25Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-qslmq-head-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:25Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}]}
{"level":"info","ts":"2026-06-11T10:25:25.468Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","name":"long-running-qslmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:25Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-qslmq-head-\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:25:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-qslmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:25:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:25:25Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-11T10:25:20Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-06-11T10:25:25.470Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"47bafafe-9a62-4870-88b0-85e113d8c5ad","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.470Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"47bafafe-9a62-4870-88b0-85e113d8c5ad"}
{"level":"error","ts":"2026-06-11T10:25:25.470Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"47bafafe-9a62-4870-88b0-85e113d8c5ad","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.488Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"3fec76d5-12ec-4856-8d4e-3e3bb915133d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:25.488Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"3fec76d5-12ec-4856-8d4e-3e3bb915133d","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:25.488Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"3fec76d5-12ec-4856-8d4e-3e3bb915133d","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:25.489Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"046d9c6f-b9ee-486d-9e90-de1de77280f1","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.489Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"046d9c6f-b9ee-486d-9e90-de1de77280f1","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.489Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8118d9f6-7324-4185-9ef5-836876dc6bb1","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.491Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4"}
{"level":"error","ts":"2026-06-11T10:25:25.491Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"4a0723ea-5732-44b1-896a-47b8972241f4","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Failed to create authentication resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.493Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"046d9c6f-b9ee-486d-9e90-de1de77280f1","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.493Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"046d9c6f-b9ee-486d-9e90-de1de77280f1","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.493Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"046d9c6f-b9ee-486d-9e90-de1de77280f1"}
{"level":"error","ts":"2026-06-11T10:25:25.493Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"046d9c6f-b9ee-486d-9e90-de1de77280f1","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.495Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8118d9f6-7324-4185-9ef5-836876dc6bb1","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.511Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"3fec76d5-12ec-4856-8d4e-3e3bb915133d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:25.511Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"3fec76d5-12ec-4856-8d4e-3e3bb915133d","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:25:25.527Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2969bd72-7d5f-4f23-ba5b-b891ab585dd6","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.527Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2969bd72-7d5f-4f23-ba5b-b891ab585dd6","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"b3d57906-b256-4e17-aca6-0870bd60d142","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:25.528Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"b3d57906-b256-4e17-aca6-0870bd60d142","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:25.528Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:25.528Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"b3d57906-b256-4e17-aca6-0870bd60d142","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:25.528Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"b3d57906-b256-4e17-aca6-0870bd60d142","seconds":589}
{"level":"info","ts":"2026-06-11T10:25:25.529Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fd3ed894-b637-47de-9c98-99fc672020c4","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:25.529Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fd3ed894-b637-47de-9c98-99fc672020c4","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:25.529Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fd3ed894-b637-47de-9c98-99fc672020c4","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:25.530Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2969bd72-7d5f-4f23-ba5b-b891ab585dd6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.530Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2969bd72-7d5f-4f23-ba5b-b891ab585dd6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.530Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2969bd72-7d5f-4f23-ba5b-b891ab585dd6"}
{"level":"error","ts":"2026-06-11T10:25:25.530Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2969bd72-7d5f-4f23-ba5b-b891ab585dd6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.532Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"fd3ed894-b637-47de-9c98-99fc672020c4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:25.541Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0716e534-04b4-4768-9019-ca207eb8f80b","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.541Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0716e534-04b4-4768-9019-ca207eb8f80b","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.544Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0716e534-04b4-4768-9019-ca207eb8f80b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.544Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0716e534-04b4-4768-9019-ca207eb8f80b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.544Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0716e534-04b4-4768-9019-ca207eb8f80b"}
{"level":"error","ts":"2026-06-11T10:25:25.544Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"0716e534-04b4-4768-9019-ca207eb8f80b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.564Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4fe1dd57-b868-478a-8690-fa3eec784c03","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.564Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4fe1dd57-b868-478a-8690-fa3eec784c03","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.568Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4fe1dd57-b868-478a-8690-fa3eec784c03","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.568Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4fe1dd57-b868-478a-8690-fa3eec784c03","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.568Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4fe1dd57-b868-478a-8690-fa3eec784c03"}
{"level":"error","ts":"2026-06-11T10:25:25.568Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4fe1dd57-b868-478a-8690-fa3eec784c03","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.608Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4145de3c-21ab-4759-8e1f-18ab78158047","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.608Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4145de3c-21ab-4759-8e1f-18ab78158047","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.615Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4145de3c-21ab-4759-8e1f-18ab78158047","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.615Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4145de3c-21ab-4759-8e1f-18ab78158047","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.615Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4145de3c-21ab-4759-8e1f-18ab78158047"}
{"level":"error","ts":"2026-06-11T10:25:25.615Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"4145de3c-21ab-4759-8e1f-18ab78158047","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.631Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6a5a2a57-6f49-45b3-b488-cb82b043d13b","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.631Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6a5a2a57-6f49-45b3-b488-cb82b043d13b","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.631Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6a5a2a57-6f49-45b3-b488-cb82b043d13b","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.651Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6a5a2a57-6f49-45b3-b488-cb82b043d13b","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.651Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6a5a2a57-6f49-45b3-b488-cb82b043d13b"}
{"level":"error","ts":"2026-06-11T10:25:25.651Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6a5a2a57-6f49-45b3-b488-cb82b043d13b","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.696Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6984d286-f97f-46b7-98f0-349b4f43a6ed","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.696Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6984d286-f97f-46b7-98f0-349b4f43a6ed","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.699Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6984d286-f97f-46b7-98f0-349b4f43a6ed","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.699Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6984d286-f97f-46b7-98f0-349b4f43a6ed","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.699Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6984d286-f97f-46b7-98f0-349b4f43a6ed"}
{"level":"error","ts":"2026-06-11T10:25:25.699Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"6984d286-f97f-46b7-98f0-349b4f43a6ed","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.860Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"84aee675-ecb1-4bfc-a457-ac7da6f155c6","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.860Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"84aee675-ecb1-4bfc-a457-ac7da6f155c6","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.863Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"84aee675-ecb1-4bfc-a457-ac7da6f155c6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:25.863Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"84aee675-ecb1-4bfc-a457-ac7da6f155c6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.863Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"84aee675-ecb1-4bfc-a457-ac7da6f155c6"}
{"level":"error","ts":"2026-06-11T10:25:25.863Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"84aee675-ecb1-4bfc-a457-ac7da6f155c6","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.972Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"766b5eef-487a-4229-a9b5-1732237fcf0a","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:25.972Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"766b5eef-487a-4229-a9b5-1732237fcf0a","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:25.972Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"766b5eef-487a-4229-a9b5-1732237fcf0a","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:25.993Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"766b5eef-487a-4229-a9b5-1732237fcf0a","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:25.993Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"766b5eef-487a-4229-a9b5-1732237fcf0a"}
{"level":"error","ts":"2026-06-11T10:25:25.993Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"766b5eef-487a-4229-a9b5-1732237fcf0a","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:26.184Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1994e56f-1e0f-416e-bfdb-8383b00f69ad","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:26.184Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1994e56f-1e0f-416e-bfdb-8383b00f69ad","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:26.192Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1994e56f-1e0f-416e-bfdb-8383b00f69ad","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:26.192Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1994e56f-1e0f-416e-bfdb-8383b00f69ad","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:26.192Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1994e56f-1e0f-416e-bfdb-8383b00f69ad"}
{"level":"error","ts":"2026-06-11T10:25:26.192Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1994e56f-1e0f-416e-bfdb-8383b00f69ad","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:26.634Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f77920ef-3614-431a-9d87-a143c506de22","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:26.634Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f77920ef-3614-431a-9d87-a143c506de22","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:26.634Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f77920ef-3614-431a-9d87-a143c506de22","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:26.649Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f77920ef-3614-431a-9d87-a143c506de22","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:26.649Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f77920ef-3614-431a-9d87-a143c506de22"}
{"level":"error","ts":"2026-06-11T10:25:26.649Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"f77920ef-3614-431a-9d87-a143c506de22","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:26.833Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1f1c93db-a20a-4dbe-b3e6-9ad116d28412","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:26.833Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1f1c93db-a20a-4dbe-b3e6-9ad116d28412","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:26.852Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1f1c93db-a20a-4dbe-b3e6-9ad116d28412","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:26.852Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1f1c93db-a20a-4dbe-b3e6-9ad116d28412","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:26.852Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1f1c93db-a20a-4dbe-b3e6-9ad116d28412"}
{"level":"error","ts":"2026-06-11T10:25:26.852Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1f1c93db-a20a-4dbe-b3e6-9ad116d28412","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:27.930Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fa286567-b259-403b-9d82-91da0c9b50e7","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:27.930Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fa286567-b259-403b-9d82-91da0c9b50e7","mode":"IntegratedOAuth","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:27.930Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fa286567-b259-403b-9d82-91da0c9b50e7","cluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:27.954Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fa286567-b259-403b-9d82-91da0c9b50e7","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:27.954Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fa286567-b259-403b-9d82-91da0c9b50e7"}
{"level":"error","ts":"2026-06-11T10:25:27.954Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fa286567-b259-403b-9d82-91da0c9b50e7","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"long-running-qslmq-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:28.133Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7f930988-a7bd-4241-8f09-b1895f89eb3f","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:28.133Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7f930988-a7bd-4241-8f09-b1895f89eb3f","rayCluster":"long-running-qslmq"}
{"level":"error","ts":"2026-06-11T10:25:28.137Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7f930988-a7bd-4241-8f09-b1895f89eb3f","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:28.137Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7f930988-a7bd-4241-8f09-b1895f89eb3f","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:28.138Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7f930988-a7bd-4241-8f09-b1895f89eb3f"}
{"level":"error","ts":"2026-06-11T10:25:28.138Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"7f930988-a7bd-4241-8f09-b1895f89eb3f","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:28.529Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec61df6e-fdc0-4a1d-991e-4e9d0131eda1","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:28.529Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec61df6e-fdc0-4a1d-991e-4e9d0131eda1","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:28.529Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec61df6e-fdc0-4a1d-991e-4e9d0131eda1","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:28.532Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec61df6e-fdc0-4a1d-991e-4e9d0131eda1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:28.532Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec61df6e-fdc0-4a1d-991e-4e9d0131eda1","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:25:28.551Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"b14d3096-3583-4374-b7ca-29813bd36329","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:28.551Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"b14d3096-3583-4374-b7ca-29813bd36329","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:28.551Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"b14d3096-3583-4374-b7ca-29813bd36329","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:28.553Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"b14d3096-3583-4374-b7ca-29813bd36329","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"a03fd751-c092-4d3d-b585-f72e68e67a12","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c4f968a1-890c-42fc-8ad6-c91e9f3ed349"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2733300b-a428-41fe-8b13-c1b719c7f44b","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"a03fd751-c092-4d3d-b585-f72e68e67a12","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2733300b-a428-41fe-8b13-c1b719c7f44b","rayCluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"reconcileID":"23d5c0e2-f037-4c6b-9fa4-91568268027d"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"a03fd751-c092-4d3d-b585-f72e68e67a12","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"e5d33310-ca23-47a7-ae13-466477d606f6","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"e5d33310-ca23-47a7-ae13-466477d606f6","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:29.061Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"e5d33310-ca23-47a7-ae13-466477d606f6","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"e5d33310-ca23-47a7-ae13-466477d606f6","seconds":585}
{"level":"info","ts":"2026-06-11T10:25:29.061Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"a03fd751-c092-4d3d-b585-f72e68e67a12","namespace":"test-ns-c6srn","otherAuthClusters":0,"action":"will delete"}
{"level":"error","ts":"2026-06-11T10:25:29.071Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2733300b-a428-41fe-8b13-c1b719c7f44b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-11T10:25:29.071Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2733300b-a428-41fe-8b13-c1b719c7f44b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:29.071Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2733300b-a428-41fe-8b13-c1b719c7f44b"}
{"level":"error","ts":"2026-06-11T10:25:29.071Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2733300b-a428-41fe-8b13-c1b719c7f44b","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-qslmq\" is forbidden: unable to create new content in namespace test-ns-c6srn because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:29.079Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"a03fd751-c092-4d3d-b585-f72e68e67a12","name":"kuberay-gateway-access","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:29.098Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"a03fd751-c092-4d3d-b585-f72e68e67a12","name":"test-ns-c6srn-long-running-qslmq","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-11T10:25:29.122Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"8f1f1b57-79ab-40d1-85bc-d818c7cddc96"}
{"level":"info","ts":"2026-06-11T10:25:29.125Z","logger":"raycluster-resource","msg":"default","name":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.125Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-qslmq","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:29.127Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"1b35d28e-7efd-4894-bb7e-0c15ffd7e6c1"}
{"level":"info","ts":"2026-06-11T10:25:29.147Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"cf38729c-aef5-4195-bd94-e44df2cbfe43"}
{"level":"info","ts":"2026-06-11T10:25:29.147Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2476a244-d76a-49c4-bbd7-f27e2fd04211","namespace":"test-ns-c6srn","clusterName":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.147Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"63b66406-0771-4aaf-8b3c-7731ade786f2","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:29.147Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"63b66406-0771-4aaf-8b3c-7731ade786f2","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-11 10:25:13 +0000 UTC","Now":"2026-06-11T10:25:29.147Z","ShutdownTime":"2026-06-11T10:35:13.000Z"}
{"level":"info","ts":"2026-06-11T10:25:29.147Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"63b66406-0771-4aaf-8b3c-7731ade786f2","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-11T10:25:29.147Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"63b66406-0771-4aaf-8b3c-7731ade786f2","seconds":585}
{"level":"info","ts":"2026-06-11T10:25:29.148Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"a03fd751-c092-4d3d-b585-f72e68e67a12","cluster":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.148Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"76da693a-46e9-4893-84f6-e0f829331aed","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:29.148Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"76da693a-46e9-4893-84f6-e0f829331aed"}
{"level":"info","ts":"2026-06-11T10:25:29.148Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"76da693a-46e9-4893-84f6-e0f829331aed","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:25:29.148Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"76da693a-46e9-4893-84f6-e0f829331aed","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:29.166Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ea41c3bc-89f5-4c1d-a04d-3d38e3cbc303","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:29.166Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ea41c3bc-89f5-4c1d-a04d-3d38e3cbc303"}
{"level":"info","ts":"2026-06-11T10:25:29.166Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ea41c3bc-89f5-4c1d-a04d-3d38e3cbc303","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:25:29.166Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"ea41c3bc-89f5-4c1d-a04d-3d38e3cbc303","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:29.190Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"2476a244-d76a-49c4-bbd7-f27e2fd04211","namespace":"test-ns-c6srn","clusterName":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:29.447Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-c6srn"},"reconcileID":"ddde542c-d9ae-456b-a5c3-00d986ce4590"}
{"level":"info","ts":"2026-06-11T10:25:29.459Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"a61ff3cf-2853-452d-a2bc-490d3dfac222","DeletionTimestamp":"2026-06-11 10:25:29 +0000 UTC"}
{"level":"error","ts":"2026-06-11T10:25:29.460Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"a61ff3cf-2853-452d-a2bc-490d3dfac222","error":"RayCluster.ray.io \"long-running-qslmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:29.460Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"a61ff3cf-2853-452d-a2bc-490d3dfac222","rayJob":"long-running-hcqjg"}
{"level":"error","ts":"2026-06-11T10:25:29.460Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"a61ff3cf-2853-452d-a2bc-490d3dfac222","error":"Post \"http:///api/jobs/long-running-hcqjg/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:29.460Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"a61ff3cf-2853-452d-a2bc-490d3dfac222","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:25:29.484Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"fe50aa37-34c9-44ad-863f-97f5cfff96da"}
{"level":"info","ts":"2026-06-11T10:25:30.514Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c6d0f14b-c01c-4c72-9fb4-3646e68228e9","namespacedName":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"}}
{"level":"info","ts":"2026-06-11T10:25:30.514Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c6d0f14b-c01c-4c72-9fb4-3646e68228e9"}
{"level":"info","ts":"2026-06-11T10:25:30.514Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c6d0f14b-c01c-4c72-9fb4-3646e68228e9","namespace":"test-ns-c6srn","authClusters":0}
{"level":"info","ts":"2026-06-11T10:25:30.514Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"c6d0f14b-c01c-4c72-9fb4-3646e68228e9","namespace":"test-ns-c6srn"}
{"level":"info","ts":"2026-06-11T10:25:30.698Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fb12ac29-0f26-467a-ac0f-89291e3378c3","namespace":"test-ns-c6srn","clusterName":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:30.698Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-qslmq","namespace":"test-ns-c6srn"},"namespace":"test-ns-c6srn","name":"long-running-qslmq","reconcileID":"fb12ac29-0f26-467a-ac0f-89291e3378c3","namespace":"test-ns-c6srn","clusterName":"long-running-qslmq"}
{"level":"info","ts":"2026-06-11T10:25:31.551Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c744753f-ec13-49a9-9b80-8c4a9f05c99a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.551Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c744753f-ec13-49a9-9b80-8c4a9f05c99a","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.551Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c744753f-ec13-49a9-9b80-8c4a9f05c99a","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:31.554Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c744753f-ec13-49a9-9b80-8c4a9f05c99a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:31.554Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"c744753f-ec13-49a9-9b80-8c4a9f05c99a","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:25:31.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"8819e397-9b67-4583-a668-992440d77465","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"8819e397-9b67-4583-a668-992440d77465","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"8819e397-9b67-4583-a668-992440d77465","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:31.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"8819e397-9b67-4583-a668-992440d77465","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:31.861Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f76f8823-90e0-4b85-b30a-e6b67b41c637","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.861Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f76f8823-90e0-4b85-b30a-e6b67b41c637","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-11T10:25:31.873Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f76f8823-90e0-4b85-b30a-e6b67b41c637"}
{"level":"info","ts":"2026-06-11T10:25:31.873Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f76f8823-90e0-4b85-b30a-e6b67b41c637","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-11T10:25:31.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f76f8823-90e0-4b85-b30a-e6b67b41c637","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-fb2jv","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:25:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:25:31.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f76f8823-90e0-4b85-b30a-e6b67b41c637","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-11T10:25:31.883Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:31.883Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","head service name":"raycluster-head-svc","namespace":"test-ns-5fsk2"}
{"level":"info","ts":"2026-06-11T10:25:31.883Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","head service URL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-11T10:25:31.883Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485"}
{"level":"info","ts":"2026-06-11T10:25:31.883Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","fail-fb2jv",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","--no-wait","--submission-id","fail-fb2jv","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","--follow","fail-fb2jv"]}
{"level":"info","ts":"2026-06-11T10:25:31.893Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-11T10:25:31.893Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.893Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-fb2jv","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-11T10:25:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-fb2jv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:25:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-11T10:25:31.893Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"007c4784-ee36-4a8b-8077-a6e37f6e6485","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:25:31.904Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"4bb3f3c9-4b94-4293-80b4-da9ef76c2f8c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.904Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"4bb3f3c9-4b94-4293-80b4-da9ef76c2f8c","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.904Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"4bb3f3c9-4b94-4293-80b4-da9ef76c2f8c","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:31.907Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"4bb3f3c9-4b94-4293-80b4-da9ef76c2f8c","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.907Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"4bb3f3c9-4b94-4293-80b4-da9ef76c2f8c"}
{"level":"error","ts":"2026-06-11T10:25:31.907Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"4bb3f3c9-4b94-4293-80b4-da9ef76c2f8c","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.912Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"9205e503-1915-45be-a8eb-46b6581f498b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.912Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"9205e503-1915-45be-a8eb-46b6581f498b","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.912Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"9205e503-1915-45be-a8eb-46b6581f498b","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:31.918Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"9205e503-1915-45be-a8eb-46b6581f498b","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.918Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"9205e503-1915-45be-a8eb-46b6581f498b"}
{"level":"error","ts":"2026-06-11T10:25:31.918Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"9205e503-1915-45be-a8eb-46b6581f498b","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.918Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"374b7f2e-89f4-4305-8e6a-075ea78a9038","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.918Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"374b7f2e-89f4-4305-8e6a-075ea78a9038","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.918Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"374b7f2e-89f4-4305-8e6a-075ea78a9038","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:31.922Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"374b7f2e-89f4-4305-8e6a-075ea78a9038","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.922Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"374b7f2e-89f4-4305-8e6a-075ea78a9038"}
{"level":"error","ts":"2026-06-11T10:25:31.922Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"374b7f2e-89f4-4305-8e6a-075ea78a9038","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"b6ba9989-3291-4f1e-83ae-6503c841bbaf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"b6ba9989-3291-4f1e-83ae-6503c841bbaf","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"b6ba9989-3291-4f1e-83ae-6503c841bbaf","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:31.933Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"b6ba9989-3291-4f1e-83ae-6503c841bbaf","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.933Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"b6ba9989-3291-4f1e-83ae-6503c841bbaf"}
{"level":"error","ts":"2026-06-11T10:25:31.933Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"b6ba9989-3291-4f1e-83ae-6503c841bbaf","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.973Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8aaa1a0b-4825-495d-9401-1eb87d52f64e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:31.973Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8aaa1a0b-4825-495d-9401-1eb87d52f64e","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:31.973Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8aaa1a0b-4825-495d-9401-1eb87d52f64e","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:31.978Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8aaa1a0b-4825-495d-9401-1eb87d52f64e","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:31.978Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8aaa1a0b-4825-495d-9401-1eb87d52f64e"}
{"level":"error","ts":"2026-06-11T10:25:31.978Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8aaa1a0b-4825-495d-9401-1eb87d52f64e","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.059Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8f6d37c8-8326-48e6-a8a1-a9b815d30d66","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:32.059Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8f6d37c8-8326-48e6-a8a1-a9b815d30d66","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:32.059Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8f6d37c8-8326-48e6-a8a1-a9b815d30d66","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:32.064Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8f6d37c8-8326-48e6-a8a1-a9b815d30d66","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.064Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8f6d37c8-8326-48e6-a8a1-a9b815d30d66"}
{"level":"error","ts":"2026-06-11T10:25:32.064Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"8f6d37c8-8326-48e6-a8a1-a9b815d30d66","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.224Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"e7d25055-c69d-43af-8b8d-a44754b9d858","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:32.224Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"e7d25055-c69d-43af-8b8d-a44754b9d858","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:32.224Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"e7d25055-c69d-43af-8b8d-a44754b9d858","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:32.229Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"e7d25055-c69d-43af-8b8d-a44754b9d858","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.229Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"e7d25055-c69d-43af-8b8d-a44754b9d858"}
{"level":"error","ts":"2026-06-11T10:25:32.229Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"e7d25055-c69d-43af-8b8d-a44754b9d858","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.485Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-c6srn"},"reconcileID":"6db3dfc4-4900-4499-b7f3-f32ee557171a"}
{"level":"info","ts":"2026-06-11T10:25:32.550Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bb3a05ea-cd5f-4e98-a97f-96e58cbb24af","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:32.550Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bb3a05ea-cd5f-4e98-a97f-96e58cbb24af","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:32.550Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bb3a05ea-cd5f-4e98-a97f-96e58cbb24af","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:32.556Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bb3a05ea-cd5f-4e98-a97f-96e58cbb24af","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.556Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bb3a05ea-cd5f-4e98-a97f-96e58cbb24af"}
{"level":"error","ts":"2026-06-11T10:25:32.556Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bb3a05ea-cd5f-4e98-a97f-96e58cbb24af","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a2f36fe-3622-44e3-981b-de842d8102ff","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:32.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a2f36fe-3622-44e3-981b-de842d8102ff","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:32.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a2f36fe-3622-44e3-981b-de842d8102ff","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:32.924Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a2f36fe-3622-44e3-981b-de842d8102ff","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:32.924Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a2f36fe-3622-44e3-981b-de842d8102ff"}
{"level":"error","ts":"2026-06-11T10:25:32.924Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a2f36fe-3622-44e3-981b-de842d8102ff","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:33.197Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bcd74934-27d9-451a-a669-86badb75e05a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:33.197Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bcd74934-27d9-451a-a669-86badb75e05a","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:33.197Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bcd74934-27d9-451a-a669-86badb75e05a","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:33.202Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bcd74934-27d9-451a-a669-86badb75e05a","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:33.202Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bcd74934-27d9-451a-a669-86badb75e05a"}
{"level":"error","ts":"2026-06-11T10:25:33.202Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"bcd74934-27d9-451a-a669-86badb75e05a","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:34.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec54e468-63af-4c94-907d-922f9230f094","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:34.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec54e468-63af-4c94-907d-922f9230f094","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:34.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec54e468-63af-4c94-907d-922f9230f094","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:34.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"ec54e468-63af-4c94-907d-922f9230f094","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:35.762Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a5383a5-8cb1-42c6-bad4-b42fb1aca80e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:35.762Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a5383a5-8cb1-42c6-bad4-b42fb1aca80e","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:35.762Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a5383a5-8cb1-42c6-bad4-b42fb1aca80e","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"error","ts":"2026-06-11T10:25:35.765Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a5383a5-8cb1-42c6-bad4-b42fb1aca80e","JobId":"fail-fb2jv","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:35.765Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a5383a5-8cb1-42c6-bad4-b42fb1aca80e"}
{"level":"error","ts":"2026-06-11T10:25:35.765Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"3a5383a5-8cb1-42c6-bad4-b42fb1aca80e","error":"Job fail-fb2jv does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-11T10:25:36.264Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"0ceac9b8-fe71-4e5e-b9f8-05e0f18c9f73","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:36.264Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"0ceac9b8-fe71-4e5e-b9f8-05e0f18c9f73","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:36.264Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"0ceac9b8-fe71-4e5e-b9f8-05e0f18c9f73","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:36.271Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"0ceac9b8-fe71-4e5e-b9f8-05e0f18c9f73","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:37.281Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"defa7952-6e0f-466f-b4aa-2cfeb3883aa3","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:37.281Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"defa7952-6e0f-466f-b4aa-2cfeb3883aa3","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:37.281Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"defa7952-6e0f-466f-b4aa-2cfeb3883aa3","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:37.284Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"defa7952-6e0f-466f-b4aa-2cfeb3883aa3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:37.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"31e5ca42-259e-4ceb-bdfe-dff0c5796c3e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:37.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"31e5ca42-259e-4ceb-bdfe-dff0c5796c3e","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:37.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"31e5ca42-259e-4ceb-bdfe-dff0c5796c3e","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:37.316Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"31e5ca42-259e-4ceb-bdfe-dff0c5796c3e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:25Z","endTime":"2026-06-11T10:25:29Z"},"jobId":"counter-kdcbj","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-11T10:25:17Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:37.316Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"31e5ca42-259e-4ceb-bdfe-dff0c5796c3e","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-11T10:25:37.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"054b098f-e739-4c71-acdc-ffab75d85e3a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:37.336Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"054b098f-e739-4c71-acdc-ffab75d85e3a","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:25:37 +0000 UTC","Now":"2026-06-11T10:25:37.336Z","ShutdownTime":"2026-06-11T10:25:37.000Z"}
{"level":"info","ts":"2026-06-11T10:25:37.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"2f250078-3176-46d4-a2b4-67d5a1898c54","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:37.586Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-5fsk2"},"reconcileID":"2f250078-3176-46d4-a2b4-67d5a1898c54","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-11 10:25:37 +0000 UTC","Now":"2026-06-11T10:25:37.586Z","ShutdownTime":"2026-06-11T10:25:37.000Z"}
{"level":"info","ts":"2026-06-11T10:25:40.886Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f1a17c66-d793-4e65-b917-224a98db4c12","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:40.886Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f1a17c66-d793-4e65-b917-224a98db4c12","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:40.886Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f1a17c66-d793-4e65-b917-224a98db4c12","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:40.889Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f1a17c66-d793-4e65-b917-224a98db4c12","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-fb2jv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-11T10:25:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:38Z","endTime":"2026-06-11T10:25:40Z"},"jobId":"fail-fb2jv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:25:38,878\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-fb2jv: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:25:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-11T10:25:40.889Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"f1a17c66-d793-4e65-b917-224a98db4c12","old JobStatus":"","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-11T10:25:40.907Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"573d9ea5-70ec-40c2-90b7-d1c4fc6e05ce","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-11T10:25:40.907Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"573d9ea5-70ec-40c2-90b7-d1c4fc6e05ce","name":"raycluster"}
{"level":"info","ts":"2026-06-11T10:25:40.907Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"573d9ea5-70ec-40c2-90b7-d1c4fc6e05ce","RayCluster":{"name":"raycluster","namespace":"test-ns-5fsk2"}}
{"level":"info","ts":"2026-06-11T10:25:40.910Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-5fsk2"},"reconcileID":"573d9ea5-70ec-40c2-90b7-d1c4fc6e05ce","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:38Z","endTime":"2026-06-11T10:25:40Z"},"jobId":"fail-fb2jv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:25:38,878\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-fb2jv: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:25:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-11T10:25:38Z","endTime":"2026-06-11T10:25:40Z"},"jobId":"fail-fb2jv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-5fsk2.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-11 10:25:38,878\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-fb2jv: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-11T10:25:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-11T10:12:15Z","stateTransitionTimes":{"ready":"2026-06-11T10:12:15Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.27","serviceIP":"10.132.0.27","podName":"raycluster-head-t5q9r","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-11T10:12:09Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-11T10:12:15Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-11T10:08:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-11T10:08:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}